react-native-image-stitcher 0.14.1 → 0.15.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (119) hide show
  1. package/CHANGELOG.md +160 -0
  2. package/README.md +35 -0
  3. package/RNImageStitcher.podspec +8 -1
  4. package/android/build.gradle +0 -16
  5. package/android/src/main/cpp/CMakeLists.txt +2 -63
  6. package/android/src/main/cpp/image_stitcher_jni.cpp +14 -0
  7. package/android/src/main/cpp/keyframe_gate_jni.cpp +13 -0
  8. package/android/src/main/java/io/imagestitcher/rn/BatchStitcher.kt +285 -3
  9. package/android/src/main/java/io/imagestitcher/rn/IncrementalStitcher.kt +180 -1162
  10. package/android/src/main/java/io/imagestitcher/rn/KeyframeGate.kt +29 -0
  11. package/android/src/main/java/io/imagestitcher/rn/RNImageStitcherPackage.kt +0 -4
  12. package/android/src/main/java/io/imagestitcher/rn/RNSARCameraView.kt +13 -64
  13. package/cpp/keyframe_gate.cpp +82 -23
  14. package/cpp/keyframe_gate.hpp +31 -2
  15. package/cpp/stitcher.cpp +208 -28
  16. package/cpp/tests/CMakeLists.txt +18 -12
  17. package/cpp/tests/keyframe_timebudget_test.cpp +65 -0
  18. package/cpp/tests/warp_guard_test.cpp +48 -0
  19. package/cpp/warp_guard.hpp +41 -0
  20. package/dist/ar/useARSession.d.ts +9 -0
  21. package/dist/ar/useARSession.js +24 -2
  22. package/dist/camera/Camera.d.ts +31 -16
  23. package/dist/camera/Camera.js +27 -4
  24. package/dist/camera/CaptureStitchStatsToast.d.ts +15 -2
  25. package/dist/camera/CaptureStitchStatsToast.js +27 -7
  26. package/dist/camera/PanoramaSettings.d.ts +10 -223
  27. package/dist/camera/PanoramaSettings.js +6 -28
  28. package/dist/camera/PanoramaSettingsBridge.d.ts +1 -24
  29. package/dist/camera/PanoramaSettingsBridge.js +3 -102
  30. package/dist/camera/PanoramaSettingsModal.js +7 -1
  31. package/dist/camera/buildPanoramaInitialSettings.d.ts +11 -0
  32. package/dist/camera/buildPanoramaInitialSettings.js +4 -0
  33. package/dist/camera/cameraErrorMessages.d.ts +32 -0
  34. package/dist/camera/cameraErrorMessages.js +53 -0
  35. package/dist/camera/selectCaptureDevice.d.ts +5 -1
  36. package/dist/camera/selectCaptureDevice.js +22 -2
  37. package/dist/camera/useCapture.js +38 -0
  38. package/dist/index.d.ts +5 -8
  39. package/dist/index.js +11 -34
  40. package/dist/stitching/incremental.d.ts +1 -117
  41. package/dist/stitching/stitchVideo.d.ts +0 -35
  42. package/dist/types.d.ts +0 -87
  43. package/ios/Sources/RNImageStitcher/IncrementalStitcher.swift +96 -674
  44. package/ios/Sources/RNImageStitcher/IncrementalStitcherBridge.swift +9 -12
  45. package/ios/Sources/RNImageStitcher/KeyframeGate.swift +14 -0
  46. package/ios/Sources/RNImageStitcher/KeyframeGateBridge.h +7 -0
  47. package/ios/Sources/RNImageStitcher/KeyframeGateBridge.mm +6 -0
  48. package/ios/Sources/RNImageStitcher/OpenCVKeyframeCollector.h +2 -2
  49. package/ios/Sources/RNImageStitcher/OpenCVKeyframeCollector.mm +3 -3
  50. package/ios/Sources/RNImageStitcher/OpenCVStitcher.h +28 -60
  51. package/ios/Sources/RNImageStitcher/OpenCVStitcher.mm +180 -921
  52. package/ios/Sources/RNImageStitcher/RNSARSession.swift +10 -35
  53. package/ios/Sources/RNImageStitcher/Stitcher.swift +84 -35
  54. package/ios/Sources/RNImageStitcher/StitcherBridge.m +13 -0
  55. package/ios/Sources/RNImageStitcher/StitcherBridge.swift +132 -5
  56. package/package.json +3 -2
  57. package/src/ar/useARSession.ts +35 -5
  58. package/src/camera/Camera.tsx +63 -24
  59. package/src/camera/CaptureStitchStatsToast.tsx +58 -14
  60. package/src/camera/PanoramaSettings.ts +16 -289
  61. package/src/camera/PanoramaSettingsBridge.ts +3 -114
  62. package/src/camera/PanoramaSettingsModal.tsx +14 -1
  63. package/src/camera/__tests__/PanoramaSettingsBridge.test.ts +3 -188
  64. package/src/camera/__tests__/buildPanoramaInitialSettings.test.ts +41 -0
  65. package/src/camera/__tests__/cameraErrorMessages.test.ts +76 -0
  66. package/src/camera/__tests__/selectCaptureDevice.test.ts +33 -0
  67. package/src/camera/buildPanoramaInitialSettings.ts +17 -0
  68. package/src/camera/cameraErrorMessages.ts +84 -0
  69. package/src/camera/selectCaptureDevice.ts +28 -3
  70. package/src/camera/useCapture.ts +44 -1
  71. package/src/index.ts +11 -40
  72. package/src/stitching/incremental.ts +3 -140
  73. package/src/stitching/stitchVideo.ts +0 -26
  74. package/src/types.ts +0 -95
  75. package/android/src/main/cpp/stitcher_jsi_install_jni.cpp +0 -227
  76. package/android/src/main/java/io/imagestitcher/rn/IncrementalFirstwinsEngine.kt +0 -1081
  77. package/android/src/main/java/io/imagestitcher/rn/StitcherJsiInstallerModule.kt +0 -103
  78. package/android/src/main/java/io/imagestitcher/rn/StitcherWorkletRuntime.kt +0 -256
  79. package/cpp/stitcher_frame_jsi.cpp +0 -214
  80. package/cpp/stitcher_frame_jsi.hpp +0 -108
  81. package/cpp/stitcher_proxy_jsi.cpp +0 -109
  82. package/cpp/stitcher_proxy_jsi.hpp +0 -46
  83. package/cpp/stitcher_worklet_dispatch.cpp +0 -103
  84. package/cpp/stitcher_worklet_dispatch.hpp +0 -71
  85. package/cpp/stitcher_worklet_registry.cpp +0 -91
  86. package/cpp/stitcher_worklet_registry.hpp +0 -146
  87. package/cpp/tests/stitcher_worklet_registry_test.cpp +0 -195
  88. package/dist/stitching/IncrementalStitcherView.d.ts +0 -41
  89. package/dist/stitching/IncrementalStitcherView.js +0 -157
  90. package/dist/stitching/StitcherWorkletRegistry.d.ts +0 -117
  91. package/dist/stitching/StitcherWorkletRegistry.js +0 -78
  92. package/dist/stitching/ensureStitcherProxyInstalled.d.ts +0 -8
  93. package/dist/stitching/ensureStitcherProxyInstalled.js +0 -81
  94. package/dist/stitching/useFrameProcessor.d.ts +0 -119
  95. package/dist/stitching/useFrameProcessor.js +0 -196
  96. package/dist/stitching/useFrameStream.d.ts +0 -34
  97. package/dist/stitching/useFrameStream.js +0 -234
  98. package/dist/stitching/useThrottledFrameProcessor.d.ts +0 -33
  99. package/dist/stitching/useThrottledFrameProcessor.js +0 -132
  100. package/ios/Sources/RNImageStitcher/OpenCVIncrementalStitcher.h +0 -474
  101. package/ios/Sources/RNImageStitcher/OpenCVIncrementalStitcher.mm +0 -1328
  102. package/ios/Sources/RNImageStitcher/OpenCVSlitScanStitcher.h +0 -103
  103. package/ios/Sources/RNImageStitcher/OpenCVSlitScanStitcher.mm +0 -3285
  104. package/ios/Sources/RNImageStitcher/RNSARWorkletRuntime.h +0 -128
  105. package/ios/Sources/RNImageStitcher/RNSARWorkletRuntime.mm +0 -313
  106. package/ios/Sources/RNImageStitcher/SaveFrameAsJpegPlugin.mm +0 -185
  107. package/ios/Sources/RNImageStitcher/StitcherFrameHostObject.h +0 -60
  108. package/ios/Sources/RNImageStitcher/StitcherFrameHostObject.mm +0 -214
  109. package/ios/Sources/RNImageStitcher/StitcherJsiInstaller.h +0 -42
  110. package/ios/Sources/RNImageStitcher/StitcherJsiInstaller.mm +0 -160
  111. package/src/stitching/IncrementalStitcherView.tsx +0 -198
  112. package/src/stitching/StitcherWorkletRegistry.ts +0 -156
  113. package/src/stitching/__tests__/StitcherWorkletRegistry.test.ts +0 -176
  114. package/src/stitching/__tests__/ensureStitcherProxyInstalled.test.ts +0 -94
  115. package/src/stitching/__tests__/useThrottledFrameProcessor.test.ts +0 -178
  116. package/src/stitching/ensureStitcherProxyInstalled.ts +0 -141
  117. package/src/stitching/useFrameProcessor.ts +0 -226
  118. package/src/stitching/useFrameStream.ts +0 -271
  119. package/src/stitching/useThrottledFrameProcessor.ts +0 -145
@@ -1,132 +0,0 @@
1
- "use strict";
2
- // SPDX-License-Identifier: Apache-2.0
3
- //
4
- // v0.9.0 Layer 2 — throttle gate over v0.8.0's `useFrameProcessor`.
5
- //
6
- // ## What this is
7
- //
8
- // A thin wrapper around `useFrameProcessor` that enforces a maximum
9
- // invocation rate (`sampleHz`) at the worklet layer. The host's
10
- // worklet fires up to `sampleHz` times per second; ticks too close
11
- // together are dropped via a `useSharedValue<number>` monotonic-time
12
- // gate inside the worklet body.
13
- //
14
- // ## When to use this (vs alternatives)
15
- //
16
- // - **`useFrameProcessor` directly** — every camera frame (~30-60 Hz).
17
- // Use for true-realtime processing that wants to see every frame.
18
- // - **`useThrottledFrameProcessor`** (this hook) — sub-frame-rate
19
- // worklet-native processing. The worklet runtime has direct
20
- // access to `frame.toArrayBuffer()`, `frame.arDepth`,
21
- // `frame.arAnchors`, and can call other vc Frame Processor plugins
22
- // (native OCR libraries, TFLite ML inference, etc.). Results
23
- // bridged to JS via `runOnJS`.
24
- // - **`useFrameStream`** (Layer 3, also in this directory) —
25
- // sub-frame-rate JS-thread consumer. The lib JPEG-encodes each
26
- // sample on the producer thread and delivers a `SampledFrame`
27
- // (file path + pose + dims) to a JS-thread callback. Use for
28
- // file-path OCR libraries (RN modules wrapping ML Kit etc.),
29
- // cloud upload, thumbnail UI.
30
- //
31
- // ## Use-case mapping (canonical)
32
- //
33
- // | Use case | Layer | Why |
34
- // |---------------------------------------|-------|----------------------------------|
35
- // | OCR via Vision.framework / ML Kit | **2** | native libs, bbox in frame coords|
36
- // | TFLite ML detection (via vc plugin) | **2** | same shape as OCR |
37
- // | LiDAR depth → 3D reconstruction | **2** | depth too large to bridge |
38
- // | Pose-only telemetry | **2** | tiny payload, no encoding needed |
39
- // | File-path OCR (RN module) | 3 | host wants a JPEG, not pixels |
40
- // | Cloud upload (sampled JPEG feed) | 3 | JPEG IS the payload |
41
- // | Live thumbnail preview UI | 3 | `<Image source={{uri: ...}}>` |
42
- //
43
- // See `docs/host-app-integration.md` § "Tier 2 + 3" for recipes.
44
- //
45
- // ## Threading
46
- //
47
- // The wrapped worklet fires on whatever runtime `useFrameProcessor`
48
- // dispatches on:
49
- // - **Non-AR mode**: vision-camera's Frame Processor runtime
50
- // (producer thread).
51
- // - **AR mode**: the lib's `RNSARWorkletRuntime` (iOS) /
52
- // worklets-core default context (Android) — fired by the AR
53
- // session's per-frame dispatch. See v0.8.0 Phase 4b.i / 4b.iii.
54
- //
55
- // Either way, the worklet MUST NOT block — the next frame's
56
- // processing is gated on this one returning. Long work belongs
57
- // behind `runOnJS` / a separate worklet runtime.
58
- //
59
- // ## Behaviour at the throttle boundary
60
- //
61
- // The hook tracks a monotonic-time shared value of "last sample time".
62
- // Each tick checks if `frame.timestamp - lastSampleMs.value >=
63
- // (1000 / sampleHz)`. If yes, the worklet body runs and the value
64
- // updates; if no, the worklet returns silently.
65
- //
66
- // Edge cases:
67
- // - First-ever tick: `lastSampleMs.value` starts at 0; first frame's
68
- // timestamp will be >> 0 → first tick always fires. Subsequent
69
- // ticks throttle as expected.
70
- // - vc v4 timestamp semantics: per the project's worklet-throttle
71
- // gotcha note, `frame.timestamp` is NOT reliably nanoseconds in
72
- // vc v4. The hook treats `frame.timestamp` as ALREADY in
73
- // milliseconds (which is what vc v4 actually delivers; the
74
- // v0.8.0 StitcherFrame contract documents this). If a future
75
- // vc version changes the unit, the throttle math here needs
76
- // re-checking.
77
- Object.defineProperty(exports, "__esModule", { value: true });
78
- exports.useThrottledFrameProcessor = useThrottledFrameProcessor;
79
- const react_native_worklets_core_1 = require("react-native-worklets-core");
80
- const useFrameProcessor_1 = require("./useFrameProcessor");
81
- /**
82
- * Throttled variant of `useFrameProcessor`. See the module
83
- * docstring for the full use-case mapping; quick version:
84
- *
85
- * ```tsx
86
- * const fp = useThrottledFrameProcessor(
87
- * (frame) => {
88
- * 'worklet';
89
- * // worklet-native OCR / ML / depth processing here
90
- * },
91
- * { sampleHz: 2 },
92
- * [],
93
- * );
94
- * return <Camera frameProcessor={fp} ... />;
95
- * ```
96
- *
97
- * @param worklet Host's frame-processor worklet. Must be
98
- * `'worklet'`-prefixed. Runs at most `sampleHz`
99
- * times per second.
100
- * @param options `{ sampleHz }` — clamped to `[0.5, 30]`.
101
- * @param deps Standard React deps array. Treated the same as
102
- * `useFrameProcessor`'s deps — when they change the
103
- * inner worklet is re-bound.
104
- *
105
- * @returns A `useFrameProcessor`-shaped processor object, pass it
106
- * to `<Camera frameProcessor={...}>`.
107
- */
108
- function useThrottledFrameProcessor(worklet, options, deps) {
109
- // Clamp + derive interval. Done outside the worklet so the
110
- // useSharedValue / useFrameProcessor hooks see stable values.
111
- const sampleHz = Math.max(0.5, Math.min(30, options.sampleHz));
112
- const minIntervalMs = 1000 / sampleHz;
113
- // Monotonic-time gate. Initialised to 0 → first tick always
114
- // fires (frame.timestamp >> 0).
115
- const lastSampleMs = (0, react_native_worklets_core_1.useSharedValue)(0);
116
- // eslint-disable-next-line react-hooks/exhaustive-deps
117
- return (0, useFrameProcessor_1.useFrameProcessor)((frame) => {
118
- 'worklet';
119
- const now = frame.timestamp;
120
- if (now - lastSampleMs.value < minIntervalMs) {
121
- return;
122
- }
123
- lastSampleMs.value = now;
124
- worklet(frame);
125
- },
126
- // The throttle interval is captured in the worklet closure; if
127
- // it changes we need to re-bind the worklet so the new
128
- // `minIntervalMs` takes effect. Same for the host's worklet
129
- // identity (so deps changes on the host side re-bind too).
130
- [minIntervalMs, worklet, ...deps]);
131
- }
132
- //# sourceMappingURL=useThrottledFrameProcessor.js.map
@@ -1,474 +0,0 @@
1
- // SPDX-License-Identifier: Apache-2.0
2
- //
3
- // OpenCVIncrementalStitcher.h
4
- //
5
- // Per-frame incremental panorama stitcher. Replaces the batch-mode
6
- // `cv::Stitcher` flow used by `OpenCVStitcher` with a streaming
7
- // pipeline: each accepted frame is matched against the previous,
8
- // warped via a RANSAC homography, and feather-blended onto a running
9
- // canvas — no end-of-capture wait.
10
- //
11
- // Why incremental:
12
- // See docs/site-content/design/2026-04-30-realtime-incremental-stitching.md
13
- // for the full motivation. TL;DR: live preview, bounded memory,
14
- // fail-fast on bad frames, no terminal BA stall.
15
- //
16
- // What this file owns:
17
- // The C++/OpenCV side of the engine — feature extraction, matching,
18
- // RANSAC, warp, feather blend. All `cv::*` types stay inside the
19
- // `.mm` impl; this header exposes only Foundation types so it can
20
- // be imported from pure Swift via the umbrella header.
21
- //
22
- // Threading:
23
- // Methods on this class are NOT thread-safe internally. The Swift
24
- // layer (`IncrementalStitcher`) owns a serial queue and
25
- // funnels all calls through it. The lock is intentional: live
26
- // capture wants ordered frame ingestion, not parallel mutation of
27
- // the canvas.
28
- //
29
-
30
- #import <Foundation/Foundation.h>
31
- #import <CoreVideo/CoreVideo.h>
32
-
33
- NS_ASSUME_NONNULL_BEGIN
34
-
35
- /// NSError domain raised by incremental stitcher errors.
36
- extern NSString *const RNImageStitcherIncrementalErrorDomain;
37
-
38
- /// Per-frame outcome — drives the JS-side UX (silent accept, subtle
39
- /// flag, explicit hint).
40
- typedef NS_ENUM(NSInteger, RLISFrameOutcome) {
41
- /// Frame accepted with high confidence. Silent UX update.
42
- RLISFrameOutcomeAcceptedHigh = 0,
43
- /// Frame accepted but match quality was middling. Show subtle
44
- /// confidence flag (yellow ring) — not an error, just informational.
45
- RLISFrameOutcomeAcceptedMedium = 1,
46
- /// Frame skipped because pose hasn't moved enough since last accept.
47
- /// Normal — operator hasn't panned past the overlap window yet.
48
- RLISFrameOutcomeSkippedTooClose = 2,
49
- /// Frame skipped because pose moved too far since last accept —
50
- /// operator panned past the overlap window before another accept.
51
- /// JS shows a "slow down" hint.
52
- RLISFrameOutcomeRejectedTooFar = 3,
53
- /// Feature matching produced too few correspondences. Scene is
54
- /// likely uniform/textureless or the frame is motion-blurred.
55
- /// JS shows a "scene too uniform" hint.
56
- RLISFrameOutcomeRejectedSceneUniform = 4,
57
- /// RANSAC homography failed or produced a degenerate transform.
58
- /// JS shows an "alignment lost — slow down" hint.
59
- RLISFrameOutcomeRejectedAlignmentLost = 5,
60
- /// Tracking state from the AR session was poor at the time of
61
- /// this frame — no point trying to incorporate it.
62
- RLISFrameOutcomeSkippedTrackingPoor = 6,
63
- /// V12.11 Step D — operator has panned BACKWARDS past the
64
- /// running max along the pan axis by more than
65
- /// `kReverseStopPx`. Engine has SKIPPED the paste; host should
66
- /// auto-finalize the capture and surface the panorama as it
67
- /// stood at the running-max position. Emitted by the
68
- /// rectilinear engine only — cylindrical engines tolerate
69
- /// reverse motion via their warp pipeline.
70
- RLISFrameOutcomeRejectedReverseDirection = 7,
71
- };
72
-
73
- /// Telemetry returned alongside each addFrame call — host can log
74
- /// these to refine threshold tuning during field testing.
75
- @interface RLISFrameTelemetry : NSObject
76
- @property (nonatomic, readonly) RLISFrameOutcome outcome;
77
- /// Estimated FoV-overlap with the previously accepted frame, in
78
- /// percent. Computed from pose-delta + intrinsics, NOT from
79
- /// matched features (which would require running the matcher
80
- /// every frame). Range [0, 100]. -1 if first frame.
81
- @property (nonatomic, readonly) double overlapPercent;
82
- /// Number of feature matches that survived ratio-test filtering.
83
- /// Zero unless the frame went through feature matching (i.e.
84
- /// passed the pose-delta gate).
85
- @property (nonatomic, readonly) NSInteger matchCount;
86
- /// Fraction of matches that survived RANSAC inlier filtering.
87
- /// Range [0, 1]. Zero unless the frame went through RANSAC.
88
- @property (nonatomic, readonly) double inlierRatio;
89
- /// Composite confidence score [0, 1].
90
- @property (nonatomic, readonly) double confidence;
91
- /// Wall-clock milliseconds the addFrame call took (end-to-end).
92
- @property (nonatomic, readonly) double processingMs;
93
- /// V12.12 — physical device orientation as detected by the engine
94
- /// from `R_panToCam` at first frame. TRUE for landscape capture
95
- /// (vertical pan), FALSE for portrait capture (horizontal pan).
96
- /// Stays at the FIRST-FRAME determination for the rest of the
97
- /// capture (orientation can't physically change without restarting
98
- /// pano). Defaults to FALSE (portrait) before first frame.
99
- ///
100
- /// JS side reads this from `IncrementalState.isLandscape` to drive
101
- /// orientation-aware UI (band overlay, dim bars). This is the
102
- /// single source of truth for orientation across the SDK + host —
103
- /// pose-derived detection (V12.6) is preferred because it works
104
- /// regardless of host orientation config (portrait-locked hosts
105
- /// suppress framebuffer rotation, so `useWindowDimensions` lies
106
- /// about the physical hold; pose data doesn't).
107
- @property (nonatomic, readonly) BOOL isLandscape;
108
-
109
- /// V12.14.9 — running max paint position along the pan axis, in
110
- /// canvas pixels. In landscape mode (`isLandscape == TRUE`) this
111
- /// is the canvas Y at which the most-recently-pasted slit ends;
112
- /// in portrait mode (`isLandscape == FALSE` = portrait+horizontal-pan
113
- /// per the two-mode spec) this is the canvas X. Zero before
114
- /// first frame is accepted. JS-side band overlay computes
115
- /// `fillRatio = paintedExtent / panExtent` to size the thumb.
116
- @property (nonatomic, readonly) NSInteger paintedExtent;
117
-
118
- /// V12.14.9 — total pan-axis extent of the canvas (the engine's
119
- /// `_canvasPanExtent` config value, default 5000). Constant for
120
- /// the lifetime of a capture. Emitted on every telemetry frame
121
- /// for symmetry with `paintedExtent`; JS uses the ratio.
122
- @property (nonatomic, readonly) NSInteger panExtent;
123
- @end
124
-
125
-
126
- /// V15 — paint-mode toggle for the slit-scan engine.
127
- /// `RLISPaintModeFirstPaintedWins` preserves the first frame's content
128
- /// (V13.0e+ default). `RLISPaintModeFeatherBlend` alpha-blends new
129
- /// content into already-painted pixels at slit boundaries (V13.0d-style
130
- /// row alpha ramp), aiming to smooth visible seams when many slits
131
- /// stack with small per-accept advance.
132
- typedef NS_ENUM(NSInteger, RLISPaintMode) {
133
- RLISPaintModeFirstPaintedWins = 0,
134
- RLISPaintModeFeatherBlend = 1,
135
- };
136
-
137
- /// V15.0c — where on the camera frame the per-accept sliver is taken
138
- /// from. For a typical landscape vertical pan tilting DOWN, the LEADING
139
- /// EDGE (new content not seen by previous frames) is at the BOTTOM of
140
- /// the camera sensor frame; for upward tilt, the leading edge is at
141
- /// the TOP. `Center` is the V13.x default (sliver from the centred
142
- /// 70% / 30% of pan-axis).
143
- typedef NS_ENUM(NSInteger, RLISSliverPosition) {
144
- RLISSliverPositionCenter = 0,
145
- RLISSliverPositionBottom = 1,
146
- RLISSliverPositionTop = 2,
147
- };
148
-
149
- /// V15 — projection toggle for the hybrid engine.
150
- /// `RLISHybridProjectionCylindrical` is the V12.x baseline; `Planar`
151
- /// uses cv::detail::PlaneWarper, well-behaved for pans under ~60°.
152
- typedef NS_ENUM(NSInteger, RLISHybridProjection) {
153
- RLISHybridProjectionCylindrical = 0,
154
- RLISHybridProjectionPlanar = 1,
155
- };
156
-
157
- /// V15.0d — source of the plane used by the slit-scan engine's V15.0b
158
- /// plane-projected stitch path. Replaces V15.0b's boolean
159
- /// `useDetectedPlane` toggle (which is kept as a deprecated alias)
160
- /// with three explicit options:
161
- ///
162
- /// • `Disabled` — no plane projection; slit-scan path runs
163
- /// (V13.x baseline + V15 refinements).
164
- /// • `ARKitDetected` — use the first vertical plane that ARKit
165
- /// finds AND whose surface normal aligns with
166
- /// the camera's view direction (filter
167
- /// threshold = `arkitPlaneAlignmentThreshold`).
168
- /// If no aligned plane is detected, the engine
169
- /// falls back to the slit-scan path silently.
170
- /// • `Virtual` — synthesize a plane from the FIRST plane-
171
- /// projected frame's camera pose:
172
- /// origin = camera_pos + virtualPlaneDepthMeters
173
- /// × camera_forward
174
- /// normal = -camera_forward
175
- /// No ARKit dependency; always available.
176
- /// Loses the "real depth" advantage of an
177
- /// ARKit-detected plane.
178
- ///
179
- /// Why both ARKit and Virtual exist:
180
- /// Field testing showed ARKit plane detection often picks the WRONG
181
- /// surface (side wall, doorframe, table edge) instead of the fixture
182
- /// face the user is scanning — producing nonsense projections (huge
183
- /// corner ray distances, 90°-rotated quads). Virtual side-steps this
184
- /// with a synthetic plane perpendicular to the camera at first frame.
185
- /// Operators can A/B between the two and pick whichever wins for
186
- /// their typical scene.
187
- typedef NS_ENUM(NSInteger, RLISPlaneSource) {
188
- RLISPlaneSourceDisabled = 0,
189
- RLISPlaneSourceARKitDetected = 1,
190
- RLISPlaneSourceVirtual = 2,
191
- };
192
-
193
- /// V15.0g — how the plane-projection helper renders each frame onto
194
- /// the canvas. Affects ARKitDetected and Virtual modes; ignored when
195
- /// planeSource = Disabled.
196
- ///
197
- /// • `Trapezoidal` (V15.0b legacy):
198
- /// Geometrically correct 3D mapping. Each camera pixel is
199
- /// raycast onto the plane and pasted at the resulting plane-
200
- /// local canvas position. When the camera tilts off-
201
- /// perpendicular, the projected camera frame becomes a
202
- /// TRAPEZOID — visually distorted (Ram observed cooler bottom
203
- /// 2.3× wider than top at 30° tilt, 2026-05-08).
204
- /// • `Rectified` (V15.0g default):
205
- /// Camera frame is pasted as a CLEAN RECTANGLE around its
206
- /// projected anchor on the canvas. Anchor is the canvas
207
- /// position of the camera CENTER raycast. Rectangle size
208
- /// depends on plane distance × pixels-per-meter. No
209
- /// trapezoidal distortion regardless of tilt angle, at the
210
- /// cost of true 3D-correctness (the camera's per-pixel
211
- /// perspective is preserved within the rectangle, but
212
- /// different tilts don't reconcile geometrically — they
213
- /// overlap with FirstPaintedWins keeping the earliest paint).
214
- ///
215
- /// Field-validate Rectified vs Trapezoidal; the right choice depends
216
- /// on the operator's typical pan range and tolerance for distortion.
217
- typedef NS_ENUM(NSInteger, RLISPlaneProjectionStyle) {
218
- RLISPlaneProjectionStyleTrapezoidal = 0,
219
- RLISPlaneProjectionStyleRectified = 1,
220
- };
221
-
222
- /// V15 stitcher config — single source of truth for which correction
223
- /// stages run in the slit-scan and hybrid engines. Each engine mode
224
- /// (`hybrid`, `slitscan-rotate`, `slitscan-both`) has a default config
225
- /// returned by `+configForMode:`; JS-side callers (settings UI, capture
226
- /// start options) override individual fields on top of the default.
227
- ///
228
- /// V13.0e+/V13.0g/V14.0a correction stages are preserved in the source;
229
- /// each is gated on the corresponding `enableX` flag. Field iteration
230
- /// happens by toggling settings, not by recompiling.
231
- @interface RLISStitcherConfig : NSObject
232
-
233
- // ── Slit shaping (slit-scan engine only) ────────────────────────────
234
-
235
- /// Fraction of the pan-axis the rectilinear slit retains per frame
236
- /// (the rest is cropped equally from both edges). Range 0.10 – 0.70.
237
- /// Default 0.30 for both slitscan modes; n/a for hybrid.
238
- @property (nonatomic) double kPanAxisFractionRect;
239
-
240
- /// Minimum pan-axis advance required before a frame is accepted.
241
- /// 0 = accept on every consumeFrame (Apple-dense slit-scan); 50 =
242
- /// V13.0g default. Default 0 for both slitscan modes; n/a for hybrid.
243
- @property (nonatomic) NSInteger kMinAcceptDeltaPx;
244
-
245
- // ── Per-stage correction toggles (slit-scan engine) ─────────────────
246
-
247
- /// V13.0e+: ORB triangulation + median-Z parallax correction.
248
- @property (nonatomic) BOOL enableTriangulation;
249
- /// V13.0g: per-accept incremental Δt accumulator on top of triangulation.
250
- @property (nonatomic) BOOL enableTriAccumulator;
251
-
252
- /// V15 new: 1D NCC perpendicular-axis wobble correction (slitscan-rotate).
253
- @property (nonatomic) BOOL enable1dNcc;
254
- /// 1D NCC search radius in pixels (5 – 30).
255
- @property (nonatomic) NSInteger nccSearchRadius1d;
256
-
257
- /// V13.0g: 2D NCC fine-alignment after triangulation.
258
- @property (nonatomic) BOOL enable2dNcc;
259
- /// V15.0d: 2D NCC search half-window in pixels. Was a hardcoded
260
- /// constexpr (V13.0g: 30, V15.0c.4: 12). Smaller = less wandering on
261
- /// repetitive textures, but easier to miss the true overlap when pose
262
- /// noise is high. Range 4 – 30. Default 12 for slitscan modes.
263
- @property (nonatomic) NSInteger nccSearchMargin2d;
264
- /// V15.0d: 2D NCC confidence threshold below which the correction
265
- /// is rejected. Was hardcoded (V13.0g: 0.6, V15.0c.4: 0.75). Higher
266
- /// = stricter — fewer false matches on repetitive textures, but more
267
- /// frames where NCC silently doesn't fire. Range 0.4 – 0.95. Default
268
- /// 0.75 for slitscan modes.
269
- @property (nonatomic) double nccConfidenceThreshold2d;
270
-
271
- /// V15.0d new (1B): EMA smoothing on 2D NCC corrections. When enabled,
272
- /// the applied correction is `α × current + (1−α) × prev` instead of
273
- /// just `current`. Dampens single-frame snaps to spurious peaks at
274
- /// the cost of a 2-frame lag. Default OFF for slitscan modes.
275
- @property (nonatomic) BOOL enableNcc2dEmaSmoothing;
276
- /// V15.0d new: EMA weight on the CURRENT-frame NCC correction (the
277
- /// remaining `1 − α` weight is on the previous correction). Range
278
- /// 0.1 – 0.9. Default 0.4 (60% prev / 40% current — heavy damping).
279
- @property (nonatomic) double ncc2dEmaAlpha;
280
-
281
- /// V15.0d new (1C): pan-axis-aware 2D NCC. When enabled, the cross-
282
- /// axis (perpendicular to the pan direction) NCC correction is
283
- /// clamped to ±`ncc2dCrossAxisLockPx`, regardless of what the search
284
- /// window size allows. Idea: 1D NCC already handles cross-axis
285
- /// wobble; 2D NCC's cross-axis search is mostly noise. Default OFF.
286
- @property (nonatomic) BOOL enableNcc2dPanAxisLock;
287
- /// V15.0d new: cross-axis clamp for the pan-axis-aware mode. Range
288
- /// 0 – 15 px. Default 5.
289
- @property (nonatomic) NSInteger ncc2dCrossAxisLockPx;
290
-
291
- /// V14.0a: RANSAC homography per slit + cv::warpPerspective.
292
- @property (nonatomic) BOOL enableRansacHomography;
293
-
294
- /// V15 new: paint mode for the slit-scan engine. Default
295
- /// FirstPaintedWins for slitscan-rotate, FeatherBlend for slitscan-both.
296
- @property (nonatomic) RLISPaintMode paintMode;
297
-
298
- /// V15.0c new: where on the camera frame the per-accept sliver is
299
- /// taken. Default Center (V13.x behaviour). Bottom = leading edge for
300
- /// typical top-to-bottom landscape pan.
301
- @property (nonatomic) RLISSliverPosition sliverPosition;
302
-
303
- /// V15.0c new: when YES, the FIRST accepted frame paints the entire
304
- /// camera frame at canvas (0, 0) instead of just the sliver. Subsequent
305
- /// frames still use the configured sliver clip. Useful with sliverPosition=
306
- /// Bottom: the first frame anchors the canvas with full-frame content,
307
- /// then leading-edge slivers extend the canvas as the camera pans.
308
- /// Default YES for slitscan-rotate / slitscan-both.
309
- @property (nonatomic) BOOL firstFrameFullFrame;
310
-
311
- /// **DEPRECATED in V15.0d** — use `planeSource` instead.
312
- ///
313
- /// V15.0b boolean toggle for the plane-projected stitch path. Kept
314
- /// as an alias for backward compat: when `planeSource` is left at
315
- /// its default (Disabled), `useDetectedPlane = YES` upgrades it to
316
- /// `ARKitDetected`. New callers should set `planeSource` directly.
317
- ///
318
- /// V15.0b semantics: if YES, the slit-scan engine projects each
319
- /// accepted frame onto a vertical plane. Composes with paint mode;
320
- /// bypasses the slit-axis 2D refinements (triangulation, 2D NCC,
321
- /// RANSAC homography) — those don't apply when the canvas is a
322
- /// real 3D plane.
323
- @property (nonatomic) BOOL useDetectedPlane;
324
-
325
- /// V15.0d new: source of the plane used by the V15.0b path. See
326
- /// `RLISPlaneSource` enum docs above for tradeoffs. Default
327
- /// Disabled for all engine modes; settings UI / capture overrides
328
- /// promote to ARKitDetected or Virtual.
329
- @property (nonatomic) RLISPlaneSource planeSource;
330
-
331
- /// V15.0d new: depth (metres) at which the synthetic plane is placed
332
- /// in front of the camera when `planeSource = Virtual`. Set the
333
- /// plane at the user's typical scan distance — too close = scene
334
- /// content gets clipped behind the plane; too far = perspective
335
- /// distortion grows. Range 0.3 – 5.0 m. Default 1.5 m.
336
- @property (nonatomic) double virtualPlaneDepthMeters;
337
-
338
- /// V15.0d new: minimum dot product between the candidate plane's
339
- /// surface normal and the camera's negative-forward direction
340
- /// (i.e. the direction the camera is facing). Used by
341
- /// `RNSARSession.didAdd` to filter ARKit-detected planes for
342
- /// `planeSource = ARKitDetected`. 1.0 = plane perfectly facing
343
- /// camera; 0.0 = plane edge-on to camera; -1.0 = facing away.
344
- /// Range 0.0 – 1.0. Default 0.6 (≈53° max angle off-camera).
345
- @property (nonatomic) double arkitPlaneAlignmentThreshold;
346
-
347
- /// V15.0g new: plane projection rendering style. See enum docs above
348
- /// for tradeoffs (Trapezoidal = 3D-correct + distorted; Rectified =
349
- /// clean-rectangle + slight 3D approximation). Ignored when
350
- /// planeSource = Disabled. Default Rectified for slit-scan modes.
351
- @property (nonatomic) RLISPlaneProjectionStyle planeProjectionStyle;
352
-
353
- // ── Hybrid-specific ─────────────────────────────────────────────────
354
-
355
- /// V15 new: projection for hybrid engine. Default Planar in V15
356
- /// (was Cylindrical in V12.x – V14.0a).
357
- @property (nonatomic) RLISHybridProjection hybridProjection;
358
-
359
- /// Build a default config for the named engine mode.
360
- /// Recognised modes: `@"hybrid"`, `@"slitscan-rotate"`,
361
- /// `@"slitscan-both"`. Backward-compat: `@"firstwins-rectilinear"`
362
- /// maps to `slitscan-rotate`; legacy `@"firstwins"` /
363
- /// `@"firstwins-zoomed"` log a deprecation warning and fall back to
364
- /// `slitscan-both`. Unrecognised modes default to `slitscan-both`.
365
- + (instancetype)configForMode:(NSString *)mode;
366
-
367
- @end
368
-
369
-
370
- /// Snapshot of the current panorama canvas. Returned by `snapshot`.
371
- @interface RLISSnapshot : NSObject
372
- /// Path to the JPEG written for this snapshot. Lives in
373
- /// `NSTemporaryDirectory()` and is overwritten on each snapshot —
374
- /// the host is expected to consume it before requesting the next.
375
- @property (nonatomic, copy, readonly) NSString *panoramaPath;
376
- @property (nonatomic, readonly) NSInteger width;
377
- @property (nonatomic, readonly) NSInteger height;
378
- @property (nonatomic, readonly) NSInteger acceptedCount;
379
- @end
380
-
381
-
382
- @interface OpenCVIncrementalStitcher : NSObject
383
-
384
- /// Initialise an engine ready to accept frames at the given compose
385
- /// resolution. `composeWidth` and `composeHeight` are the dimensions
386
- /// each ingested ARFrame is scaled to before feature extraction —
387
- /// 720p (1280×720 landscape) is the design-doc default. Smaller =
388
- /// faster + less memory at the cost of feature density.
389
- ///
390
- /// `canvasWidth` and `canvasHeight` size the pre-allocated panorama
391
- /// canvas (CV_8UC3). Pick generously to avoid clipping long pans.
392
- /// Defaults if 0/0 passed: 4800×1600 (≈23 MB). The first accepted
393
- /// frame is placed in the canvas centre so growth in either pan
394
- /// direction is symmetric.
395
- - (instancetype)initWithComposeWidth:(NSInteger)composeWidth
396
- composeHeight:(NSInteger)composeHeight
397
- canvasWidth:(NSInteger)canvasWidth
398
- canvasHeight:(NSInteger)canvasHeight
399
- featherPx:(NSInteger)featherPx
400
- frameRotationDegrees:(NSInteger)frameRotationDegrees NS_DESIGNATED_INITIALIZER;
401
-
402
- - (instancetype)init NS_UNAVAILABLE;
403
-
404
- /// V15 — set the per-stage correction config. Should be called once
405
- /// after init, before any `ingestPixelBuffer:` call. If never called,
406
- /// the engine uses a default equivalent to
407
- /// `+[RLISStitcherConfig configForMode:@"hybrid"]`.
408
- - (void)setConfig:(RLISStitcherConfig *)config;
409
-
410
- /// Try to incorporate `pixelBuffer` into the running panorama.
411
- ///
412
- /// V6 (pose-driven): the engine builds the warp homography
413
- /// `H = T · K · M · R_first⁻¹ · R_new · M · K⁻¹` directly from the
414
- /// ARKit camera quaternion and intrinsics passed alongside the
415
- /// frame. No feature extraction, no matching, no RANSAC — the
416
- /// alignment is geometrically exact for the rotational pans that
417
- /// dominate handheld panoramas. `M = diag(1, -1, -1)` converts
418
- /// ARKit's (Y-up, -Z forward) camera frame to OpenCV's standard
419
- /// (Y-down, +Z forward) frame.
420
- ///
421
- /// Pose-delta gating still uses (yaw, pitch, fov*Degrees) to skip
422
- /// frames outside the overlap window before any warp work runs.
423
- ///
424
- /// `trackingPoor` should be YES when the AR session reports
425
- /// non-tracking state at the time of this frame; the engine then
426
- /// skips immediately with `RLISFrameOutcomeSkippedTrackingPoor`.
427
- - (RLISFrameTelemetry *)ingestPixelBuffer:(CVPixelBufferRef)pixelBuffer
428
- qx:(double)qx
429
- qy:(double)qy
430
- qz:(double)qz
431
- qw:(double)qw
432
- tx:(double)tx
433
- ty:(double)ty
434
- tz:(double)tz
435
- fx:(double)fx
436
- fy:(double)fy
437
- cx:(double)cx
438
- cy:(double)cy
439
- imageWidth:(NSInteger)imageWidth
440
- imageHeight:(NSInteger)imageHeight
441
- yaw:(double)yaw
442
- pitch:(double)pitch
443
- fovHorizDegrees:(double)fovHorizDegrees
444
- fovVertDegrees:(double)fovVertDegrees
445
- trackingPoor:(BOOL)trackingPoor
446
- NS_SWIFT_NAME(ingest(pixelBuffer:qx:qy:qz:qw:tx:ty:tz:fx:fy:cx:cy:imageWidth:imageHeight:yaw:pitch:fovHorizDegrees:fovVertDegrees:trackingPoor:));
447
-
448
- /// Snapshot the current panorama as a JPEG (overwriting any previous
449
- /// snapshot file). Cheap enough to call after each accepted frame
450
- /// for live-preview UX. Returns nil with `error` populated if the
451
- /// snapshot failed (disk full, permission, etc.).
452
- - (nullable RLISSnapshot *)snapshotWithJpegQuality:(NSInteger)quality
453
- error:(NSError **)error;
454
-
455
- /// Final write at end of capture — same shape as `snapshot` but
456
- /// written to `outputPath` (caller-controlled location). Includes
457
- /// a tight crop to the actual panorama bounds (no trailing canvas
458
- /// black). After this call, the canvas is reset; the engine is
459
- /// ready for a fresh capture without re-init.
460
- - (nullable RLISSnapshot *)finalizeAtPath:(NSString *)outputPath
461
- jpegQuality:(NSInteger)quality
462
- error:(NSError **)error;
463
-
464
- /// Reset state so the engine can begin a new capture. Called
465
- /// automatically by `finalizeAtPath:` and on construction.
466
- - (void)reset;
467
-
468
- /// Frames accepted into the panorama since `reset`. Read-only;
469
- /// monotonically increasing within a capture.
470
- @property (nonatomic, readonly) NSInteger acceptedCount;
471
-
472
- @end
473
-
474
- NS_ASSUME_NONNULL_END