react-native-image-stitcher 0.15.1 → 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +147 -1
- package/README.md +116 -5
- package/android/src/main/cpp/image_stitcher_jni.cpp +107 -11
- package/android/src/main/java/io/imagestitcher/rn/BatchStitcher.kt +223 -1
- package/android/src/main/java/io/imagestitcher/rn/IncrementalStitcher.kt +87 -30
- package/android/src/main/java/io/imagestitcher/rn/KeyframeGate.kt +1 -1
- package/android/src/main/java/io/imagestitcher/rn/RNSARSession.kt +14 -8
- package/android/src/main/java/io/imagestitcher/rn/ar/YuvImageConverter.kt +39 -1
- package/cpp/crop_quad.cpp +162 -0
- package/cpp/crop_quad.hpp +163 -0
- package/cpp/stitcher.cpp +651 -55
- package/cpp/stitcher.hpp +10 -0
- package/cpp/warp_guard.hpp +212 -0
- package/dist/camera/Camera.d.ts +196 -12
- package/dist/camera/Camera.js +629 -35
- package/dist/camera/CameraView.js +62 -5
- package/dist/camera/CaptureCountdownOverlay.d.ts +70 -0
- package/dist/camera/CaptureCountdownOverlay.js +239 -0
- package/dist/camera/CaptureFrameCounterOverlay.d.ts +58 -0
- package/dist/camera/CaptureFrameCounterOverlay.js +142 -0
- package/dist/camera/CaptureMemoryPill.d.ts +9 -1
- package/dist/camera/CaptureMemoryPill.js +3 -3
- package/dist/camera/CapturePreview.js +2 -1
- package/dist/camera/CaptureStatusOverlay.d.ts +11 -4
- package/dist/camera/CaptureStatusOverlay.js +22 -5
- package/dist/camera/CaptureThumbnailStrip.js +2 -1
- package/dist/camera/LateralMotionModal.d.ts +85 -0
- package/dist/camera/LateralMotionModal.js +134 -0
- package/dist/camera/PanHowToOverlay.d.ts +76 -0
- package/dist/camera/PanHowToOverlay.js +222 -0
- package/dist/camera/PanoramaSettings.d.ts +8 -6
- package/dist/camera/PanoramaSettings.js +26 -5
- package/dist/camera/PanoramaSettingsModal.js +4 -4
- package/dist/camera/RectCropPreview.d.ts +161 -0
- package/dist/camera/RectCropPreview.js +480 -0
- package/dist/camera/RotateToLandscapePrompt.d.ts +87 -0
- package/dist/camera/RotateToLandscapePrompt.js +138 -0
- package/dist/camera/buildPanoramaInitialSettings.d.ts +19 -2
- package/dist/camera/buildPanoramaInitialSettings.js +9 -0
- package/dist/camera/cameraErrorMessages.d.ts +30 -1
- package/dist/camera/cameraErrorMessages.js +26 -10
- package/dist/camera/cameraGuidanceCopy.d.ts +87 -0
- package/dist/camera/cameraGuidanceCopy.js +80 -0
- package/dist/camera/captureCountdown.d.ts +52 -0
- package/dist/camera/captureCountdown.js +76 -0
- package/dist/camera/captureWarnings.d.ts +90 -0
- package/dist/camera/captureWarnings.js +108 -0
- package/dist/camera/classifyStitchError.d.ts +30 -0
- package/dist/camera/classifyStitchError.js +42 -0
- package/dist/camera/cropGeometry.d.ts +136 -0
- package/dist/camera/cropGeometry.js +223 -0
- package/dist/camera/displayDecodeImageProps.d.ts +25 -0
- package/dist/camera/displayDecodeImageProps.js +29 -0
- package/dist/camera/guidanceGraphics.d.ts +58 -0
- package/dist/camera/guidanceGraphics.js +280 -0
- package/dist/camera/guidanceTokens.d.ts +54 -0
- package/dist/camera/guidanceTokens.js +58 -0
- package/dist/camera/panModeGate.d.ts +54 -0
- package/dist/camera/panModeGate.js +62 -0
- package/dist/camera/pickCaptureFormat.d.ts +71 -0
- package/dist/camera/pickCaptureFormat.js +85 -0
- package/dist/camera/stitchDebugInfo.d.ts +27 -0
- package/dist/camera/stitchDebugInfo.js +55 -0
- package/dist/camera/usePanMotion.d.ts +250 -0
- package/dist/camera/usePanMotion.js +451 -0
- package/dist/index.d.ts +24 -3
- package/dist/index.js +33 -2
- package/dist/stitching/computeInscribedRect.d.ts +40 -0
- package/dist/stitching/computeInscribedRect.js +55 -0
- package/dist/stitching/cropQuad.d.ts +78 -0
- package/dist/stitching/cropQuad.js +116 -0
- package/dist/stitching/incremental.d.ts +45 -0
- package/ios/Sources/RNImageStitcher/IncrementalStitcher.swift +56 -8
- package/ios/Sources/RNImageStitcher/KeyframeGate.swift +2 -2
- package/ios/Sources/RNImageStitcher/OpenCVKeyframeCollector.mm +48 -5
- package/ios/Sources/RNImageStitcher/OpenCVStitcher.h +27 -0
- package/ios/Sources/RNImageStitcher/OpenCVStitcher.mm +191 -7
- package/ios/Sources/RNImageStitcher/RNSARSession.swift +25 -1
- package/ios/Sources/RNImageStitcher/Stitcher.swift +34 -1
- package/ios/Sources/RNImageStitcher/StitcherBridge.m +5 -0
- package/ios/Sources/RNImageStitcher/StitcherBridge.swift +56 -0
- package/package.json +5 -1
- package/src/camera/Camera.tsx +994 -47
- package/src/camera/CameraView.tsx +75 -5
- package/src/camera/CaptureCountdownOverlay.tsx +272 -0
- package/src/camera/CaptureFrameCounterOverlay.tsx +183 -0
- package/src/camera/CaptureMemoryPill.tsx +17 -3
- package/src/camera/CapturePreview.tsx +5 -0
- package/src/camera/CaptureStatusOverlay.tsx +35 -7
- package/src/camera/CaptureThumbnailStrip.tsx +4 -0
- package/src/camera/LateralMotionModal.tsx +199 -0
- package/src/camera/PanHowToOverlay.tsx +246 -0
- package/src/camera/PanoramaSettings.ts +34 -11
- package/src/camera/PanoramaSettingsModal.tsx +4 -4
- package/src/camera/RectCropPreview.tsx +820 -0
- package/src/camera/RotateToLandscapePrompt.tsx +188 -0
- package/src/camera/buildPanoramaInitialSettings.ts +30 -1
- package/src/camera/cameraErrorMessages.ts +39 -2
- package/src/camera/cameraGuidanceCopy.ts +145 -0
- package/src/camera/captureCountdown.ts +83 -0
- package/src/camera/captureWarnings.ts +190 -0
- package/src/camera/classifyStitchError.ts +68 -0
- package/src/camera/cropGeometry.ts +268 -0
- package/src/camera/displayDecodeImageProps.ts +25 -0
- package/src/camera/guidanceGraphics.tsx +347 -0
- package/src/camera/guidanceTokens.ts +57 -0
- package/src/camera/panModeGate.ts +81 -0
- package/src/camera/pickCaptureFormat.ts +130 -0
- package/src/camera/stitchDebugInfo.ts +71 -0
- package/src/camera/usePanMotion.ts +667 -0
- package/src/index.ts +66 -3
- package/src/stitching/computeInscribedRect.ts +81 -0
- package/src/stitching/cropQuad.ts +167 -0
- package/src/stitching/incremental.ts +45 -0
- package/cpp/tests/CMakeLists.txt +0 -104
- package/cpp/tests/README.md +0 -86
- package/cpp/tests/keyframe_timebudget_test.cpp +0 -65
- package/cpp/tests/pose_test.cpp +0 -74
- package/cpp/tests/stitcher_frame_data_test.cpp +0 -132
- package/cpp/tests/stubs/jsi/jsi.h +0 -33
- package/cpp/tests/stubs/react-native-worklets-core/WKTJsiWorklet.h +0 -34
- package/cpp/tests/warp_guard_test.cpp +0 -48
- package/src/camera/__tests__/PanoramaSettingsBridge.test.ts +0 -190
- package/src/camera/__tests__/bandThumbRotation.test.ts +0 -120
- package/src/camera/__tests__/buildPanoramaInitialSettings.test.ts +0 -160
- package/src/camera/__tests__/cameraErrorMessages.test.ts +0 -76
- package/src/camera/__tests__/homeIndicatorEdge.test.ts +0 -116
- package/src/camera/__tests__/lowMemDevice.test.ts +0 -52
- package/src/camera/__tests__/selectCaptureDevice.test.ts +0 -210
- package/src/camera/__tests__/useContentRotation.test.ts +0 -89
- package/src/camera/__tests__/useOrientationDrift.test.ts +0 -169
- package/src/stitching/__tests__/subscribeIncrementalState.refine.test.ts +0 -276
- package/src/stitching/__tests__/useStitcherWorklet.test.ts +0 -202
package/cpp/stitcher.hpp
CHANGED
|
@@ -92,6 +92,7 @@ enum class StitchErrorCode : int32_t {
|
|
|
92
92
|
ComposeResizeFailed = 104,
|
|
93
93
|
WarpFailed = 105,
|
|
94
94
|
EmptyPanorama = 106,
|
|
95
|
+
LowQualityStitch = 107, // post-stitch validator: disjoint/fragmented output
|
|
95
96
|
InvalidArgument = 200,
|
|
96
97
|
UnknownCvException = 300,
|
|
97
98
|
};
|
|
@@ -225,6 +226,15 @@ struct StitchResult {
|
|
|
225
226
|
// StitchConfig::stitchMode iff the fallback ran. Defaults to
|
|
226
227
|
// Panorama for back-compat in code paths that don't set it.
|
|
227
228
|
StitchMode stitchModeUsed = StitchMode::Panorama;
|
|
229
|
+
|
|
230
|
+
// 2026-06-14 (DEV overlay) — a human-readable, machine-parseable trace of
|
|
231
|
+
// the choices the stitcher actually made for THIS output, surfaced on the
|
|
232
|
+
// preview in __DEV__ so the user can see HOW a panorama was built without
|
|
233
|
+
// reading logcat/Console. Semicolon-separated `key=value` pairs, e.g.
|
|
234
|
+
// "pipe=manual;warp=spherical;route=batch;seam=graphcut;blend=multiband"
|
|
235
|
+
// Empty on builds that don't populate it (back-compat). iOS marshals it up
|
|
236
|
+
// to the JS finalize dict; Android leaves it in the log for now.
|
|
237
|
+
std::string debugSummary;
|
|
228
238
|
};
|
|
229
239
|
|
|
230
240
|
|
package/cpp/warp_guard.hpp
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
// SPDX-License-Identifier: Apache-2.0
|
|
2
2
|
#pragma once
|
|
3
3
|
#include <cstdint>
|
|
4
|
+
#include <cmath>
|
|
4
5
|
|
|
5
6
|
// ─────────────────────────────────────────────────────────────────────
|
|
6
7
|
// Warp-canvas size guard — shared by the warp pre-pass (which decides
|
|
@@ -21,6 +22,23 @@ namespace retailens {
|
|
|
21
22
|
// frame, and blending several would jetsam-OOM the app. 100 megapixels.
|
|
22
23
|
constexpr int64_t kMaxWarpPixels = 100LL * 1000LL * 1000LL;
|
|
23
24
|
|
|
25
|
+
// Max size of the CUMULATIVE blend canvas — the bounding box over every
|
|
26
|
+
// positioned warp rect (corner + size) that `cv::detail::Blender::prepare`
|
|
27
|
+
// allocates as its CV_16SC3 accumulator (~6 bytes/px) plus a CV_8U mask
|
|
28
|
+
// and, for MultiBand, Laplacian-pyramid overhead (~1.5-2× on top). This
|
|
29
|
+
// is a DIFFERENT axis from kMaxWarpPixels: a degenerate homography can
|
|
30
|
+
// shift ONE frame's corner to a huge offset so the union spans gigapixels
|
|
31
|
+
// while every individual frame's extent still passes the per-frame guard.
|
|
32
|
+
// Guarding the union before prepare() is what actually stops crash B (the
|
|
33
|
+
// 51 MB → 3.7 GB single-pan blow-up).
|
|
34
|
+
//
|
|
35
|
+
// 50 MP sizing: a valid 360° cylindrical canvas is ~9 MP (2π·~1200 px
|
|
36
|
+
// focal × ~1200 px tall) and real field-log panoramas are ~1.3 MP, so
|
|
37
|
+
// 50 MP is ~5× headroom over the widest legitimate pano (zero false
|
|
38
|
+
// positives) while 50 MP × (6 + 1) bytes + pyramid overhead ≈ 500-600 MB
|
|
39
|
+
// peak — comfortably under the 6 GB-class pre-stitch headroom.
|
|
40
|
+
constexpr int64_t kMaxCanvasPixels = 50LL * 1000LL * 1000LL;
|
|
41
|
+
|
|
24
42
|
// True if a warp ROI of `width`×`height` px is degenerate: non-positive
|
|
25
43
|
// in either dimension, or strictly larger than `maxPixels` (so a canvas
|
|
26
44
|
// exactly at the limit is still allowed).
|
|
@@ -38,4 +56,198 @@ inline bool warpRoiExceedsGuard(int width, int height,
|
|
|
38
56
|
return pixels > maxPixels;
|
|
39
57
|
}
|
|
40
58
|
|
|
59
|
+
// True if the cumulative blend-canvas of `width`×`height` px is degenerate:
|
|
60
|
+
// non-positive in either dimension, or strictly larger than `maxPixels`
|
|
61
|
+
// (so a canvas exactly at the limit is still allowed). Same int64 area
|
|
62
|
+
// math as warpRoiExceedsGuard — the union of a degenerate corner offset is
|
|
63
|
+
// exactly the case where int32 area would overflow. Takes int64 dims
|
|
64
|
+
// because the union is computed in int64 (a degenerate corner can exceed
|
|
65
|
+
// the int32 range on its own).
|
|
66
|
+
inline bool canvasExceedsGuard(int64_t width, int64_t height,
|
|
67
|
+
int64_t maxPixels = kMaxCanvasPixels) {
|
|
68
|
+
if (width <= 0 || height <= 0) {
|
|
69
|
+
return true;
|
|
70
|
+
}
|
|
71
|
+
// width/height are already bounded by the caller's union math, but cap
|
|
72
|
+
// the multiply defensively: if either exceeds ~3 G the product overflows
|
|
73
|
+
// int64, and such a dimension is degenerate by any measure.
|
|
74
|
+
if (width > 3'000'000'000LL || height > 3'000'000'000LL) {
|
|
75
|
+
return true;
|
|
76
|
+
}
|
|
77
|
+
return width * height > maxPixels;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
// ─────────────────────────────────────────────────────────────────────
|
|
81
|
+
// RAM-aware output-canvas budget (the wide-pan blend-OOM fix).
|
|
82
|
+
//
|
|
83
|
+
// Distinct from the guards above: a VALID but wide pan produces a large
|
|
84
|
+
// union canvas, and the BATCH + MultiBand blend peak scales with it (on a
|
|
85
|
+
// 6 GB device a ~70 MP union hit ~2.97 GB RSS and was lmkd-killed mid-
|
|
86
|
+
// blend). Rather than REJECT a valid capture, we cap the canvas to a
|
|
87
|
+
// memory budget by reducing compose scale — yielding a slightly-lower-res
|
|
88
|
+
// but COMPLETE panorama. The two functions below are the OpenCV-free,
|
|
89
|
+
// unit-testable core of that cap (the warp/resize itself lives in
|
|
90
|
+
// stitcher.cpp and is on-device-only).
|
|
91
|
+
//
|
|
92
|
+
// kBlendBytesPerUnionPx was back-solved from the on-device capture-14
|
|
93
|
+
// failure: (2970 MB peak − ~330 MB baseline) / 70.7 MP ≈ 37.4 B per union
|
|
94
|
+
// pixel. Round up to 38 for headroom. kBudgetCeilMP (42) keeps a 6 GB
|
|
95
|
+
// device's predicted peak (~1.9 GB) under its lmkd death point while
|
|
96
|
+
// staying ≤ kMaxCanvasPixels (50 MP) so the degenerate-canvas guard above
|
|
97
|
+
// never fires on a cap-eligible pan. kBudgetFloorMP (12) is > the widest
|
|
98
|
+
// VALID 360° panorama (~9 MP), so a normal pano is provably never capped.
|
|
99
|
+
constexpr double kBlendBytesPerUnionPx = 38.0;
|
|
100
|
+
constexpr double kBlendRamFraction = 0.30;
|
|
101
|
+
constexpr double kBudgetFloorMP = 12.0;
|
|
102
|
+
constexpr double kBudgetCeilMP = 42.0;
|
|
103
|
+
|
|
104
|
+
// Output-canvas megapixel budget for a device with `totalRamMB` of RAM.
|
|
105
|
+
// Monotonic-nondecreasing in RAM, clamped to [floor, ceil]. A non-
|
|
106
|
+
// positive/sentinel RAM falls to the floor (the caller should resolve a
|
|
107
|
+
// -1 sentinel to an assumed RAM before calling, but never get a <=0
|
|
108
|
+
// budget regardless).
|
|
109
|
+
inline double composeCanvasBudgetMP(double totalRamMB) {
|
|
110
|
+
const double raw = (totalRamMB * kBlendRamFraction) / kBlendBytesPerUnionPx;
|
|
111
|
+
if (raw < kBudgetFloorMP) return kBudgetFloorMP;
|
|
112
|
+
if (raw > kBudgetCeilMP) return kBudgetCeilMP;
|
|
113
|
+
return raw;
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
// Linear downscale factor that brings a `canvasMP`-megapixel canvas down to
|
|
117
|
+
// `budgetMP`. Canvas area scales with factor², so factor = sqrt(budget /
|
|
118
|
+
// canvas), clamped to [0.2, 1.0]: never UPSCALES (≤ 1.0 — a canvas already
|
|
119
|
+
// within budget returns 1.0, a no-op), and never collapses below 0.2 (a
|
|
120
|
+
// canvas still over budget after 0.2× is degenerate, which the separate
|
|
121
|
+
// canvasExceedsGuard net catches on its own axis). Returns 1.0 when either
|
|
122
|
+
// input is non-positive (no div-by-zero; matches a "nothing to cap" no-op).
|
|
123
|
+
inline double canvasDownscaleForBudget(double canvasMP, double budgetMP) {
|
|
124
|
+
if (canvasMP <= 0.0 || budgetMP <= 0.0 || canvasMP <= budgetMP) {
|
|
125
|
+
return 1.0;
|
|
126
|
+
}
|
|
127
|
+
double factor = std::sqrt(budgetMP / canvasMP);
|
|
128
|
+
if (factor < 0.2) factor = 0.2;
|
|
129
|
+
if (factor > 1.0) factor = 1.0;
|
|
130
|
+
return factor;
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
// Seam-finder downscale aspect, re-capped against the WARPED image size.
|
|
134
|
+
//
|
|
135
|
+
// The GraphCut seam finder must run at ~`seamMp` megapixels per image (what
|
|
136
|
+
// cv::Stitcher's seam_est_resol targets) or its per-pixel max-flow graph
|
|
137
|
+
// blows up — a wide-pan capture whose warped images spanned a 19 MP canvas
|
|
138
|
+
// OOM-killed the app because the seam images were multi-MP, not 0.1 MP.
|
|
139
|
+
//
|
|
140
|
+
// The caller's `inputAspect` is derived from the INPUT frame size, but the
|
|
141
|
+
// resize it feeds is applied to the WARPED images, which can be many× larger
|
|
142
|
+
// (the warp expands a ~0.3 MP frame across the whole canvas). So re-cap the
|
|
143
|
+
// aspect so the LARGEST warped frame (`maxWarpedMp`) downscales to ≤ seamMp.
|
|
144
|
+
// Never RAISES the aspect (only tightens it); a no-op when the warped images
|
|
145
|
+
// are already ≤ seamMp or the inputs are degenerate.
|
|
146
|
+
inline double cappedSeamAspect(double inputAspect, double maxWarpedMp,
|
|
147
|
+
double seamMp) {
|
|
148
|
+
if (seamMp <= 0.0 || maxWarpedMp <= seamMp) {
|
|
149
|
+
return inputAspect;
|
|
150
|
+
}
|
|
151
|
+
const double capped = std::sqrt(seamMp / maxWarpedMp);
|
|
152
|
+
return (capped < inputAspect) ? capped : inputAspect;
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
// ─────────────────────────────────────────────────────────────────────
|
|
156
|
+
// Issue 3 — post-stitch disjointness check (pure).
|
|
157
|
+
//
|
|
158
|
+
// The confidence filter drops frames that don't register, but nothing
|
|
159
|
+
// validated the OUTPUT: a frame that survived confidence yet landed
|
|
160
|
+
// geometrically disconnected shows up as a separate blob in the coverage
|
|
161
|
+
// mask ("disjointed image frames in the output"). Given the largest
|
|
162
|
+
// connected component's area, the total covered area, and the frame count,
|
|
163
|
+
// decide whether a MEANINGFUL fraction of coverage lies OUTSIDE the main
|
|
164
|
+
// blob — i.e. the frames didn't fuse into one panorama. Pure so the
|
|
165
|
+
// threshold is unit-testable; the OpenCV connected-components extraction
|
|
166
|
+
// (which feeds these areas) lives in stitcher.cpp's validateStitchOutput.
|
|
167
|
+
//
|
|
168
|
+
// Conservative by design: a normal panorama is ONE connected blob
|
|
169
|
+
// (fragmentFraction ≈ 0), so the 0.15 default never trips on it; a whole
|
|
170
|
+
// disconnected frame in a few-frame pan easily exceeds 15 % of coverage.
|
|
171
|
+
constexpr double kMaxStitchFragmentFraction = 0.15;
|
|
172
|
+
|
|
173
|
+
inline bool stitchOutputIsDisjoint(
|
|
174
|
+
double largestComponentArea, double totalCoveredArea, int numFrames,
|
|
175
|
+
double maxFragmentFraction = kMaxStitchFragmentFraction) {
|
|
176
|
+
if (numFrames < 2) return false;
|
|
177
|
+
if (totalCoveredArea <= 0.0 || largestComponentArea <= 0.0) return false;
|
|
178
|
+
const double fragmentFraction =
|
|
179
|
+
1.0 - (largestComponentArea / totalCoveredArea);
|
|
180
|
+
return fragmentFraction > maxFragmentFraction;
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
// Coverage-to-canvas UTILIZATION guard — the "black canvas" failure. When
|
|
184
|
+
// BundleAdjusterRay mis-places a weak boundary frame, PlaneWarper throws it
|
|
185
|
+
// far off-axis so the union canvas balloons and the real content clusters in
|
|
186
|
+
// one corner. That is a single coherent blob, so `stitchOutputIsDisjoint`
|
|
187
|
+
// (fragmentFraction ≈ 0) PASSES it — yet it's garbage. Guard the ratio of
|
|
188
|
+
// covered pixels to total panorama pixels instead. A valid pano (cropped to
|
|
189
|
+
// its coverage downstream) fills well above this; a marooned-corner canvas is
|
|
190
|
+
// only a percent or two. The 50 MP `canvasExceedsGuard` catches gigapixel
|
|
191
|
+
// blowups; this catches the moderate 12–50 MP band it leaves open.
|
|
192
|
+
constexpr double kMinStitchUtilization = 0.10;
|
|
193
|
+
|
|
194
|
+
inline bool stitchOutputUnderutilized(
|
|
195
|
+
double totalCoveredArea, double canvasArea, int numFrames,
|
|
196
|
+
double minUtilization = kMinStitchUtilization) {
|
|
197
|
+
if (numFrames < 2) return false;
|
|
198
|
+
if (canvasArea <= 0.0 || totalCoveredArea <= 0.0) return false;
|
|
199
|
+
return (totalCoveredArea / canvasArea) < minUtilization;
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
// ─────────────────────────────────────────────────────────────────────
|
|
203
|
+
// Issue 6 — headroom-based memory gating (pure).
|
|
204
|
+
//
|
|
205
|
+
// We CANNOT measure the stitch's own allocation apart from the shared
|
|
206
|
+
// process RSS (OpenCV uses malloc; there's no per-library accounting). So
|
|
207
|
+
// rather than a flat device-scaled RSS ceiling — which a memory-heavy HOST
|
|
208
|
+
// app trips even when the stitch itself is small — we reason about HEADROOM:
|
|
209
|
+
// estimate the per-process kill ceiling and gate on whether the stitch's
|
|
210
|
+
// INCREMENTAL demand fits on top of the CURRENT process footprint.
|
|
211
|
+
|
|
212
|
+
// Estimated per-process memory ceiling (MB) before the OS (iOS jetsam /
|
|
213
|
+
// Android lmkd) kills the app, as a fraction of total device RAM. Anchored
|
|
214
|
+
// to the iPhone 16 Pro (8 GB) observed jetsam at ~3.38 GB ⇒ ~0.42. Floored
|
|
215
|
+
// so tiny (2 GB) devices still get a sane budget.
|
|
216
|
+
constexpr double kProcessLimitFraction = 0.42;
|
|
217
|
+
constexpr double kProcessBudgetFloorMB = 900.0;
|
|
218
|
+
|
|
219
|
+
inline double perProcessMemoryBudgetMB(double totalRamMB) {
|
|
220
|
+
const double raw = totalRamMB * kProcessLimitFraction;
|
|
221
|
+
return (raw < kProcessBudgetFloorMB) ? kProcessBudgetFloorMB : raw;
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
// Smallest streaming-stitch peak we insist on having room for (one warped
|
|
225
|
+
// frame + the CV_16SC3 accumulator + masks at compose resolution).
|
|
226
|
+
// Conservative.
|
|
227
|
+
constexpr double kMinStreamStitchMB = 350.0;
|
|
228
|
+
|
|
229
|
+
// Early pre-stitch gate: abort BEFORE loading frames ONLY when the process
|
|
230
|
+
// is already so close to its ceiling that even a minimal streaming stitch
|
|
231
|
+
// won't fit on top of the current footprint. A true last resort — scoped to
|
|
232
|
+
// the stitch's MINIMAL incremental demand, not a flat device ceiling — so a
|
|
233
|
+
// heavy host app with headroom remaining still proceeds.
|
|
234
|
+
inline bool stitchExceedsMinimalHeadroom(double currentRssMB,
|
|
235
|
+
double totalRamMB) {
|
|
236
|
+
return currentRssMB + kMinStreamStitchMB
|
|
237
|
+
> perProcessMemoryBudgetMB(totalRamMB);
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
// Comfortable free headroom (MB) below which we prefer the STREAM+feather
|
|
241
|
+
// path over BATCH (graphcut+multiband), whose blend peak can spike far above
|
|
242
|
+
// STREAM's. Used as an ADDITIONAL routing trigger alongside the fixed
|
|
243
|
+
// canvas/held-set MP thresholds — it only ever makes routing MORE
|
|
244
|
+
// conservative (more likely STREAM), never less, so it can't cause an OOM
|
|
245
|
+
// that the fixed thresholds would have avoided.
|
|
246
|
+
constexpr double kBatchHeadroomMB = 1000.0;
|
|
247
|
+
|
|
248
|
+
inline bool lowBatchHeadroom(double currentRssMB, double totalRamMB) {
|
|
249
|
+
return (perProcessMemoryBudgetMB(totalRamMB) - currentRssMB)
|
|
250
|
+
< kBatchHeadroomMB;
|
|
251
|
+
}
|
|
252
|
+
|
|
41
253
|
} // namespace retailens
|
package/dist/camera/Camera.d.ts
CHANGED
|
@@ -44,7 +44,12 @@ import type { DrawableFrameProcessor, ReadonlyFrameProcessor } from 'react-nativ
|
|
|
44
44
|
import { type CaptureHeaderProps } from './CaptureHeader';
|
|
45
45
|
import { type CapturePreviewAction } from './CapturePreview';
|
|
46
46
|
import { type CaptureThumbnailItem } from './CaptureThumbnailStrip';
|
|
47
|
+
import { type CaptureStatusPhase } from './CaptureStatusOverlay';
|
|
48
|
+
import { type PanoramaPropOverrides } from './buildPanoramaInitialSettings';
|
|
47
49
|
import { type DeviceOrientation } from './useDeviceOrientation';
|
|
50
|
+
import { type PanMode } from './panModeGate';
|
|
51
|
+
import { type GuidanceCopy } from './cameraGuidanceCopy';
|
|
52
|
+
import { type CaptureWarning } from './captureWarnings';
|
|
48
53
|
export type CaptureSource = 'ar' | 'non-ar';
|
|
49
54
|
/**
|
|
50
55
|
* v0.13.2 — which capture sources the host ALLOWS. A constraint on top
|
|
@@ -63,24 +68,43 @@ export type Blender = 'multiband' | 'feather';
|
|
|
63
68
|
export type SeamFinder = 'graphcut' | 'skip';
|
|
64
69
|
export type Warper = 'plane' | 'cylindrical' | 'spherical';
|
|
65
70
|
/**
|
|
66
|
-
* Result emitted via `onCapture`. Discriminated union keyed on
|
|
67
|
-
* `
|
|
68
|
-
*
|
|
71
|
+
* Result emitted via `onCapture`. Discriminated union keyed FIRST on
|
|
72
|
+
* `ok` (success vs. failure) and then on `type` (photo vs. panorama), so a
|
|
73
|
+
* host handles EVERY capture outcome — success, degraded success, and
|
|
74
|
+
* failure — through this one callback.
|
|
69
75
|
*
|
|
70
|
-
*
|
|
71
|
-
*
|
|
72
|
-
*
|
|
73
|
-
*
|
|
74
|
-
*
|
|
75
|
-
*
|
|
76
|
-
*
|
|
76
|
+
* ## v0.16 — unified success/failure + warnings (BREAKING)
|
|
77
|
+
*
|
|
78
|
+
* Previously `onCapture` fired only on success and carried no `ok` field;
|
|
79
|
+
* failures went *solely* to `onError`. Hosts therefore had no single place
|
|
80
|
+
* to learn whether a capture succeeded, and no programmatic signal that a
|
|
81
|
+
* stitch was *degraded* (e.g. most frames dropped). Now:
|
|
82
|
+
*
|
|
83
|
+
* - `onCapture` ALWAYS fires once per capture attempt, with `ok:true`
|
|
84
|
+
* (output present) or `ok:false` (carrying the `CameraError`).
|
|
85
|
+
* - both success and failure carry `warnings: CaptureWarning[]` — non-fatal
|
|
86
|
+
* quality signals (e.g. `LOW_FRAME_UTILIZATION` when <70 % of captured
|
|
87
|
+
* frames survived, `LATERAL_DRIFT_FINALIZE` when item-6 stopped early).
|
|
88
|
+
* - `onError` STILL fires on failure too (an unchanged mirror), so existing
|
|
89
|
+
* error handling keeps working.
|
|
90
|
+
*
|
|
91
|
+
* Migration: gate on `ok` before reading `uri`/`width`/`height` —
|
|
92
|
+
* `if (!result.ok) { handle(result.error); return; }`.
|
|
93
|
+
*
|
|
94
|
+
* Identifier `CameraCaptureResult` (vs. the SDK's existing `CaptureResult`
|
|
95
|
+
* from `../types`) is intentional — the existing CaptureResult shape has
|
|
96
|
+
* SDK-specific fields that don't belong in the public RN library's surface.
|
|
77
97
|
*/
|
|
78
98
|
export type CameraCaptureResult = {
|
|
99
|
+
ok: true;
|
|
79
100
|
type: 'photo';
|
|
80
101
|
uri: string;
|
|
81
102
|
width: number;
|
|
82
103
|
height: number;
|
|
104
|
+
/** Non-fatal quality signals (empty when none). */
|
|
105
|
+
warnings: CaptureWarning[];
|
|
83
106
|
} | {
|
|
107
|
+
ok: true;
|
|
84
108
|
type: 'panorama';
|
|
85
109
|
uri: string;
|
|
86
110
|
width: number;
|
|
@@ -99,12 +123,57 @@ export type CameraCaptureResult = {
|
|
|
99
123
|
* cv::Stitcher at finalize).
|
|
100
124
|
*/
|
|
101
125
|
stitchModeResolved?: 'panorama' | 'scans';
|
|
126
|
+
/**
|
|
127
|
+
* 2026-06-14 (DEV overlay) — semicolon-separated `key=value` trace of the
|
|
128
|
+
* stitcher's runtime choices (pipe/warp/route/seam/blend) for this
|
|
129
|
+
* output. Shown on the preview in __DEV__. iOS only for now.
|
|
130
|
+
*/
|
|
131
|
+
debugSummary?: string;
|
|
132
|
+
/**
|
|
133
|
+
* 2026-06-15 (iOS) — keyframe JPEG paths used for this stitch, so the
|
|
134
|
+
* preview can re-stitch them on demand via `refinePanorama` (the
|
|
135
|
+
* high-level tab). iOS only; undefined elsewhere.
|
|
136
|
+
*/
|
|
137
|
+
keyframePaths?: string[];
|
|
138
|
+
/**
|
|
139
|
+
* 2026-06-15 (iOS) — orientation this stitch baked in. The on-demand
|
|
140
|
+
* high-level re-stitch passes it back so it matches the manual output's
|
|
141
|
+
* rotation (not the raw sensor landscape). iOS only.
|
|
142
|
+
*/
|
|
143
|
+
captureOrientation?: string;
|
|
144
|
+
/** Non-fatal quality signals (empty when none). */
|
|
145
|
+
warnings: CaptureWarning[];
|
|
146
|
+
} | {
|
|
147
|
+
ok: false;
|
|
148
|
+
/** Which capture path failed. */
|
|
149
|
+
type: 'photo' | 'panorama';
|
|
150
|
+
/** The classified failure (same object handed to `onError`). */
|
|
151
|
+
error: CameraError;
|
|
152
|
+
/** Any warnings gathered before the failure (usually empty). */
|
|
153
|
+
warnings: CaptureWarning[];
|
|
102
154
|
};
|
|
155
|
+
/**
|
|
156
|
+
* The success-panorama variant of {@link CameraCaptureResult} — the exact
|
|
157
|
+
* shape stashed for the crop editor and re-emitted (with adjusted dims) once
|
|
158
|
+
* the user crops. Narrowed so the crop-confirm spread keeps `uri`/`width`/
|
|
159
|
+
* `height`/`ok` without a cast.
|
|
160
|
+
*/
|
|
161
|
+
export type PanoramaCaptureResult = Extract<CameraCaptureResult, {
|
|
162
|
+
ok: true;
|
|
163
|
+
type: 'panorama';
|
|
164
|
+
}>;
|
|
103
165
|
/**
|
|
104
166
|
* Errors surfaced via `onError`. Classified codes so consumers can
|
|
105
167
|
* branch on the kind of failure (toast vs retry vs report).
|
|
106
168
|
*/
|
|
107
|
-
export type CameraErrorCode = 'CAMERA_PERMISSION_DENIED' | 'CAMERA_DEVICE_UNAVAILABLE' | 'PHOTO_CAPTURE_FAILED' | 'PANORAMA_START_FAILED' | 'PANORAMA_FINALIZE_FAILED' | 'STITCH_NEED_MORE_IMGS' | 'STITCH_HOMOGRAPHY_FAIL' | 'STITCH_CAMERA_PARAMS_FAIL'
|
|
169
|
+
export type CameraErrorCode = 'CAMERA_PERMISSION_DENIED' | 'CAMERA_DEVICE_UNAVAILABLE' | 'PHOTO_CAPTURE_FAILED' | 'PANORAMA_START_FAILED' | 'PANORAMA_FINALIZE_FAILED' | 'STITCH_NEED_MORE_IMGS' | 'STITCH_HOMOGRAPHY_FAIL' | 'STITCH_CAMERA_PARAMS_FAIL'
|
|
170
|
+
/**
|
|
171
|
+
* v0.16 — the native post-stitch validator rejected the output: the
|
|
172
|
+
* panorama came out disjoint / fragmented / wildly mis-proportioned
|
|
173
|
+
* (frames didn't connect into one coherent image). Recoverable by
|
|
174
|
+
* re-capturing, so it carries "try again" copy.
|
|
175
|
+
*/
|
|
176
|
+
| 'STITCH_LOW_QUALITY' | 'STITCH_OOM' | 'OUTPUT_WRITE_FAILED'
|
|
108
177
|
/**
|
|
109
178
|
* Vision-camera surfaced a runtime error that isn't a known
|
|
110
179
|
* transient lifecycle event (those are swallowed inside the SDK's
|
|
@@ -156,6 +225,19 @@ export interface CameraProps {
|
|
|
156
225
|
defaultRegistrationResolMP?: number;
|
|
157
226
|
/** Forward-looking — see above. */
|
|
158
227
|
defaultSeamEstimationResolMP?: number;
|
|
228
|
+
/**
|
|
229
|
+
* v0.16 — pass the whole stitcher config as a JSON object instead of the
|
|
230
|
+
* individual `default*` props above (canonical field names: `warperType` /
|
|
231
|
+
* `blenderType` / `seamFinderType` / `stitchMode` /
|
|
232
|
+
* `enableMaxInscribedRectCrop`). Partial; any field set here wins over the
|
|
233
|
+
* matching flat prop. Runtime ⚙️-panel edits still override at capture time. */
|
|
234
|
+
stitcher?: PanoramaPropOverrides['stitcher'];
|
|
235
|
+
/**
|
|
236
|
+
* v0.16 — pass the whole frame-gate config as a JSON object (canonical field
|
|
237
|
+
* names: `mode` / `maxKeyframes` / `overlapThreshold` / `maxKeyframeIntervalMs`
|
|
238
|
+
* / `flow`). Partial; `flow` is deep-merged. Wins over the flat `default*`
|
|
239
|
+
* props. */
|
|
240
|
+
frameSelection?: PanoramaPropOverrides['frameSelection'];
|
|
159
241
|
/**
|
|
160
242
|
* Crop strategy for the stitched panorama. `false` (default) keeps the
|
|
161
243
|
* bounding-rect of non-black pixels, which preserves all stitched
|
|
@@ -248,12 +330,19 @@ export interface CameraProps {
|
|
|
248
330
|
* decisively cancels the capture (`incremental.cancel()`) and
|
|
249
331
|
* surfaces `OrientationDriftModal` to explain what happened.
|
|
250
332
|
*
|
|
333
|
+
* v0.16 adds `'lateral-drift'`: the user moved the phone perpendicular to
|
|
334
|
+
* the pan arrow before enough frames were captured to stitch. Rather than
|
|
335
|
+
* finalize into a misleading "need more images" error, the SDK abandons the
|
|
336
|
+
* capture and surfaces the `LateralMotionModal` with "follow the arrow"
|
|
337
|
+
* copy. (A lateral drift AFTER enough frames still finalizes what was
|
|
338
|
+
* captured and fires `onCapture` with a `LATERAL_DRIFT_FINALIZE` warning.)
|
|
339
|
+
*
|
|
251
340
|
* Hosts use this callback to clean up their own state (e.g., reset
|
|
252
341
|
* a wizard step, log telemetry, surface their own retry UX in
|
|
253
342
|
* addition to the SDK's built-in modal). No `onCapture` will fire
|
|
254
343
|
* for an abandoned capture.
|
|
255
344
|
*/
|
|
256
|
-
onCaptureAbandoned?: (reason: 'orientation-drift') => void;
|
|
345
|
+
onCaptureAbandoned?: (reason: 'orientation-drift' | 'lateral-drift') => void;
|
|
257
346
|
/**
|
|
258
347
|
* v0.13.0 — flash (torch) state. Controlled-or-uncontrolled.
|
|
259
348
|
*
|
|
@@ -495,6 +584,83 @@ export interface CameraProps {
|
|
|
495
584
|
* CHANGELOG.)
|
|
496
585
|
*/
|
|
497
586
|
frameProcessor?: ReadonlyFrameProcessor | DrawableFrameProcessor;
|
|
587
|
+
/**
|
|
588
|
+
* Which device holds the non-AR panorama capture accepts.
|
|
589
|
+
*
|
|
590
|
+
* - `'vertical'` (DEFAULT) — LANDSCAPE-only, top→bottom pan. Starting a
|
|
591
|
+
* panorama in portrait is BLOCKED behind the rotate-to-landscape
|
|
592
|
+
* prompt (item 2); the capture starts the instant they rotate to
|
|
593
|
+
* landscape (either way up).
|
|
594
|
+
* - `'horizontal'` — PORTRAIT-only, left→right pan. Starting in
|
|
595
|
+
* landscape is BLOCKED behind the rotate-to-portrait prompt; capture
|
|
596
|
+
* starts on rotating to portrait (either way up).
|
|
597
|
+
* - `'both'` — landscape OR portrait; the rotate gate never fires, the
|
|
598
|
+
* user captures in whichever hold they're already in.
|
|
599
|
+
*
|
|
600
|
+
* **BREAKING (since the previous release accepted both holds ungated):**
|
|
601
|
+
* the default is now `'vertical'`. Hosts that want left→right (portrait)
|
|
602
|
+
* panoramas use `panMode='horizontal'` (portrait-only) or `'both'`. See
|
|
603
|
+
* CHANGELOG.
|
|
604
|
+
*/
|
|
605
|
+
panMode?: PanMode;
|
|
606
|
+
/**
|
|
607
|
+
* Master switch for the in-capture pan-guidance surfaces (rotate
|
|
608
|
+
* prompt, pan how-to overlay, too-fast pill, blinking countdown).
|
|
609
|
+
* Default `true`. Set `false` to suppress all of them (the lateral-
|
|
610
|
+
* drift FINALIZE behaviour and the crop preview are governed by their
|
|
611
|
+
* own props, not this flag).
|
|
612
|
+
*/
|
|
613
|
+
panGuidance?: boolean;
|
|
614
|
+
/**
|
|
615
|
+
* Optional hard recording-TIME ceiling for a non-AR panorama, in
|
|
616
|
+
* milliseconds, used as a SAFETY cap alongside the primary keyframe-count
|
|
617
|
+
* auto-stop. The default capture now finalizes when the configured
|
|
618
|
+
* keyframe count is reached (see the frame counter HUD), so this is `0`
|
|
619
|
+
* (disabled) by default. Set it to a positive value to ALSO cap the
|
|
620
|
+
* recording by wall-clock time; when > 0 a blinking countdown (item 5)
|
|
621
|
+
* shows the seconds remaining and the capture auto-finalizes at 0.
|
|
622
|
+
*
|
|
623
|
+
* v0.16 — default changed `9000` → `0` (time cap is now opt-in; the
|
|
624
|
+
* keyframe-count stop is the default UX).
|
|
625
|
+
*/
|
|
626
|
+
maxPanDurationMs?: number;
|
|
627
|
+
/**
|
|
628
|
+
* Gyro rate (rad/s) above which the pan is flagged "moving too fast"
|
|
629
|
+
* (item 4 — the transient amber pill). Optional; forwards to
|
|
630
|
+
* `usePanMotion`'s `warnMaxRadPerSec` (default 1.0 rad/s there).
|
|
631
|
+
*/
|
|
632
|
+
panTooFastThreshold?: number;
|
|
633
|
+
/**
|
|
634
|
+
* Cross-pan (lateral) drift budget in CENTIMETRES (item 6). Once the
|
|
635
|
+
* operator's integrated sideways translation exceeds this for the
|
|
636
|
+
* hook's grace window, the capture FINALIZES what was captured and a
|
|
637
|
+
* one-button popup explains why. Default `5`. `0` disables the
|
|
638
|
+
* lateral-drift stop entirely.
|
|
639
|
+
*/
|
|
640
|
+
lateralBudgetCm?: number;
|
|
641
|
+
/**
|
|
642
|
+
* Show the draggable-quad crop editor after a panorama finalizes, BEFORE
|
|
643
|
+
* emitting it via `onCapture`. Default `false`. When `true`, the user
|
|
644
|
+
* drags 4 corners over the stitched result; confirming crops in place
|
|
645
|
+
* (perspective-rectify when the quad isn't axis-aligned), "Use original"
|
|
646
|
+
* emits the un-cropped panorama, "Retake" discards it. Takes precedence
|
|
647
|
+
* over {@link showPreview}.
|
|
648
|
+
*/
|
|
649
|
+
rectCrop?: boolean;
|
|
650
|
+
/**
|
|
651
|
+
* Show a plain review screen after a panorama finalizes — the stitched
|
|
652
|
+
* image with [Retake] / [Confirm] and NO crop box. Default `false`.
|
|
653
|
+
* Ignored when {@link rectCrop} is on (the crop editor is itself the
|
|
654
|
+
* preview). With both off, `onCapture` fires immediately with no UI.
|
|
655
|
+
*/
|
|
656
|
+
showPreview?: boolean;
|
|
657
|
+
/**
|
|
658
|
+
* Copy overrides for every guidance string (rotate prompt, pan hint,
|
|
659
|
+
* too-fast warning, lateral-stop popup, crop buttons). Partial —
|
|
660
|
+
* unspecified keys fall back to {@link DEFAULT_GUIDANCE_COPY}. Hosts
|
|
661
|
+
* localise or re-word the whole guidance surface in one place here.
|
|
662
|
+
*/
|
|
663
|
+
guidanceCopy?: Partial<GuidanceCopy>;
|
|
498
664
|
}
|
|
499
665
|
/**
|
|
500
666
|
* The public `<Camera>` component.
|
|
@@ -545,5 +711,23 @@ declare function isSideEdge(edge: HomeIndicatorEdge): boolean;
|
|
|
545
711
|
export declare const _homeIndicatorEdgeForTests: typeof homeIndicatorEdge;
|
|
546
712
|
/** @internal test-only — see `isSideEdge`. */
|
|
547
713
|
export declare const _isSideEdgeForTests: typeof isSideEdge;
|
|
714
|
+
/**
|
|
715
|
+
* cameraShouldUnmount — whether the live camera (<CameraView> /
|
|
716
|
+
* <ARCameraView>) should be UNMOUNTED (replaced by the placeholder) this
|
|
717
|
+
* render rather than mounted.
|
|
718
|
+
*
|
|
719
|
+
* True while a camera-switch transition or AR-support probe is in flight,
|
|
720
|
+
* OR during the stitch (statusPhase==='stitching'). The stitching case is
|
|
721
|
+
* the V12.14.8 OOM fix: unmounting frees vision-camera's AVCaptureSession +
|
|
722
|
+
* preview buffers (~150-250 MB) BEFORE the memory-heavy stitch, so the
|
|
723
|
+
* live-camera footprint and the stitch peak never coexist and jetsam (iOS)
|
|
724
|
+
* / lmkd (Android) don't OOM-kill the app.
|
|
725
|
+
*
|
|
726
|
+
* Pure + exported for test — the lib's jest config can't mount <Camera>,
|
|
727
|
+
* so this boolean is the unit-testable core of the OOM render gate.
|
|
728
|
+
*/
|
|
729
|
+
declare function cameraShouldUnmount(inFlightTransition: boolean, arSupportPending: boolean, statusPhase: CaptureStatusPhase): boolean;
|
|
730
|
+
/** @internal test-only — see `cameraShouldUnmount`. */
|
|
731
|
+
export declare const _cameraShouldUnmountForTests: typeof cameraShouldUnmount;
|
|
548
732
|
export {};
|
|
549
733
|
//# sourceMappingURL=Camera.d.ts.map
|