react-native-image-stitcher 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +96 -0
- package/LICENSE +201 -0
- package/NOTICE +21 -0
- package/README.md +189 -0
- package/RNImageStitcher.podspec +76 -0
- package/android/build.gradle +224 -0
- package/android/src/main/AndroidManifest.xml +3 -0
- package/android/src/main/cpp/CMakeLists.txt +124 -0
- package/android/src/main/cpp/image_stitcher_jni.cpp +145 -0
- package/android/src/main/cpp/keyframe_gate_jni.cpp +204 -0
- package/android/src/main/java/io/imagestitcher/rn/BatchStitcher.kt +426 -0
- package/android/src/main/java/io/imagestitcher/rn/IncrementalFirstwinsEngine.kt +960 -0
- package/android/src/main/java/io/imagestitcher/rn/IncrementalStitcher.kt +2371 -0
- package/android/src/main/java/io/imagestitcher/rn/KeyframeGate.kt +256 -0
- package/android/src/main/java/io/imagestitcher/rn/QualityChecker.kt +167 -0
- package/android/src/main/java/io/imagestitcher/rn/RNImageStitcherPackage.kt +39 -0
- package/android/src/main/java/io/imagestitcher/rn/RNSARCameraView.kt +558 -0
- package/android/src/main/java/io/imagestitcher/rn/RNSARCameraViewManager.kt +35 -0
- package/android/src/main/java/io/imagestitcher/rn/RNSARSession.kt +784 -0
- package/android/src/main/java/io/imagestitcher/rn/ar/BackgroundRenderer.kt +176 -0
- package/android/src/main/java/io/imagestitcher/rn/ar/ShaderUtil.kt +67 -0
- package/android/src/main/java/io/imagestitcher/rn/ar/YuvImageConverter.kt +201 -0
- package/cpp/ar_frame_pose.h +63 -0
- package/cpp/keyframe_gate.cpp +927 -0
- package/cpp/keyframe_gate.hpp +240 -0
- package/cpp/stitcher.cpp +2207 -0
- package/cpp/stitcher.hpp +275 -0
- package/dist/ar/useARSession.d.ts +102 -0
- package/dist/ar/useARSession.js +133 -0
- package/dist/camera/ARCameraView.d.ts +93 -0
- package/dist/camera/ARCameraView.js +170 -0
- package/dist/camera/Camera.d.ts +134 -0
- package/dist/camera/Camera.js +688 -0
- package/dist/camera/CameraShutter.d.ts +80 -0
- package/dist/camera/CameraShutter.js +237 -0
- package/dist/camera/CameraView.d.ts +65 -0
- package/dist/camera/CameraView.js +117 -0
- package/dist/camera/CaptureControlsBar.d.ts +87 -0
- package/dist/camera/CaptureControlsBar.js +82 -0
- package/dist/camera/CaptureHeader.d.ts +62 -0
- package/dist/camera/CaptureHeader.js +81 -0
- package/dist/camera/CapturePreview.d.ts +70 -0
- package/dist/camera/CapturePreview.js +188 -0
- package/dist/camera/CaptureStatusOverlay.d.ts +75 -0
- package/dist/camera/CaptureStatusOverlay.js +326 -0
- package/dist/camera/CaptureThumbnailStrip.d.ts +87 -0
- package/dist/camera/CaptureThumbnailStrip.js +177 -0
- package/dist/camera/IncrementalPanGuide.d.ts +83 -0
- package/dist/camera/IncrementalPanGuide.js +267 -0
- package/dist/camera/PanoramaBandOverlay.d.ts +107 -0
- package/dist/camera/PanoramaBandOverlay.js +399 -0
- package/dist/camera/PanoramaConfirmModal.d.ts +57 -0
- package/dist/camera/PanoramaConfirmModal.js +128 -0
- package/dist/camera/PanoramaGuidance.d.ts +79 -0
- package/dist/camera/PanoramaGuidance.js +246 -0
- package/dist/camera/PanoramaSettingsModal.d.ts +311 -0
- package/dist/camera/PanoramaSettingsModal.js +611 -0
- package/dist/camera/ViewportCropOverlay.d.ts +46 -0
- package/dist/camera/ViewportCropOverlay.js +67 -0
- package/dist/camera/useCapture.d.ts +111 -0
- package/dist/camera/useCapture.js +160 -0
- package/dist/camera/useDeviceOrientation.d.ts +48 -0
- package/dist/camera/useDeviceOrientation.js +131 -0
- package/dist/camera/useVideoCapture.d.ts +79 -0
- package/dist/camera/useVideoCapture.js +151 -0
- package/dist/index.d.ts +26 -0
- package/dist/index.js +39 -0
- package/dist/quality/normaliseOrientation.d.ts +36 -0
- package/dist/quality/normaliseOrientation.js +62 -0
- package/dist/quality/runQualityCheck.d.ts +41 -0
- package/dist/quality/runQualityCheck.js +98 -0
- package/dist/sensors/useIMUTranslationGate.d.ts +70 -0
- package/dist/sensors/useIMUTranslationGate.js +235 -0
- package/dist/stitching/IncrementalStitcherView.d.ts +41 -0
- package/dist/stitching/IncrementalStitcherView.js +157 -0
- package/dist/stitching/incremental.d.ts +930 -0
- package/dist/stitching/incremental.js +133 -0
- package/dist/stitching/stitchFrames.d.ts +55 -0
- package/dist/stitching/stitchFrames.js +56 -0
- package/dist/stitching/stitchVideo.d.ts +119 -0
- package/dist/stitching/stitchVideo.js +57 -0
- package/dist/stitching/useIncrementalJSDriver.d.ts +74 -0
- package/dist/stitching/useIncrementalJSDriver.js +199 -0
- package/dist/stitching/useIncrementalStitcher.d.ts +58 -0
- package/dist/stitching/useIncrementalStitcher.js +172 -0
- package/dist/types.d.ts +58 -0
- package/dist/types.js +15 -0
- package/ios/Package.swift +72 -0
- package/ios/Sources/RNImageStitcher/ARCameraViewManager.m +33 -0
- package/ios/Sources/RNImageStitcher/ARCameraViewManager.swift +40 -0
- package/ios/Sources/RNImageStitcher/ARSessionBridge.m +55 -0
- package/ios/Sources/RNImageStitcher/ARSessionBridge.swift +149 -0
- package/ios/Sources/RNImageStitcher/IncrementalStitcher.swift +2727 -0
- package/ios/Sources/RNImageStitcher/IncrementalStitcherBridge.m +85 -0
- package/ios/Sources/RNImageStitcher/IncrementalStitcherBridge.swift +625 -0
- package/ios/Sources/RNImageStitcher/KeyframeGate.swift +328 -0
- package/ios/Sources/RNImageStitcher/KeyframeGateBridge.h +141 -0
- package/ios/Sources/RNImageStitcher/KeyframeGateBridge.mm +278 -0
- package/ios/Sources/RNImageStitcher/OpenCVIncrementalStitcher.h +473 -0
- package/ios/Sources/RNImageStitcher/OpenCVIncrementalStitcher.mm +1326 -0
- package/ios/Sources/RNImageStitcher/OpenCVKeyframeCollector.h +97 -0
- package/ios/Sources/RNImageStitcher/OpenCVKeyframeCollector.mm +296 -0
- package/ios/Sources/RNImageStitcher/OpenCVSlitScanStitcher.h +103 -0
- package/ios/Sources/RNImageStitcher/OpenCVSlitScanStitcher.mm +3285 -0
- package/ios/Sources/RNImageStitcher/OpenCVStitcher.h +238 -0
- package/ios/Sources/RNImageStitcher/OpenCVStitcher.mm +1880 -0
- package/ios/Sources/RNImageStitcher/QualityChecker.swift +252 -0
- package/ios/Sources/RNImageStitcher/QualityCheckerBridge.m +26 -0
- package/ios/Sources/RNImageStitcher/QualityCheckerBridge.swift +72 -0
- package/ios/Sources/RNImageStitcher/RNSARCameraView.swift +114 -0
- package/ios/Sources/RNImageStitcher/RNSARSession.swift +1111 -0
- package/ios/Sources/RNImageStitcher/Stitcher.swift +243 -0
- package/ios/Sources/RNImageStitcher/StitcherBridge.m +28 -0
- package/ios/Sources/RNImageStitcher/StitcherBridge.swift +246 -0
- package/package.json +73 -0
- package/react-native.config.js +34 -0
- package/scripts/opencv-version.txt +1 -0
- package/scripts/postinstall-fetch-binaries.js +286 -0
- package/src/ar/useARSession.ts +210 -0
- package/src/camera/.gitkeep +0 -0
- package/src/camera/ARCameraView.tsx +256 -0
- package/src/camera/Camera.tsx +1053 -0
- package/src/camera/CameraShutter.tsx +292 -0
- package/src/camera/CameraView.tsx +157 -0
- package/src/camera/CaptureControlsBar.tsx +204 -0
- package/src/camera/CaptureHeader.tsx +184 -0
- package/src/camera/CapturePreview.tsx +318 -0
- package/src/camera/CaptureStatusOverlay.tsx +391 -0
- package/src/camera/CaptureThumbnailStrip.tsx +277 -0
- package/src/camera/IncrementalPanGuide.tsx +328 -0
- package/src/camera/PanoramaBandOverlay.tsx +498 -0
- package/src/camera/PanoramaConfirmModal.tsx +206 -0
- package/src/camera/PanoramaGuidance.tsx +327 -0
- package/src/camera/PanoramaSettingsModal.tsx +1357 -0
- package/src/camera/ViewportCropOverlay.tsx +81 -0
- package/src/camera/useCapture.ts +279 -0
- package/src/camera/useDeviceOrientation.ts +140 -0
- package/src/camera/useVideoCapture.ts +236 -0
- package/src/index.ts +53 -0
- package/src/quality/.gitkeep +0 -0
- package/src/quality/normaliseOrientation.ts +79 -0
- package/src/quality/runQualityCheck.ts +131 -0
- package/src/sensors/useIMUTranslationGate.ts +347 -0
- package/src/stitching/.gitkeep +0 -0
- package/src/stitching/IncrementalStitcherView.tsx +198 -0
- package/src/stitching/incremental.ts +1021 -0
- package/src/stitching/stitchFrames.ts +88 -0
- package/src/stitching/stitchVideo.ts +153 -0
- package/src/stitching/useIncrementalJSDriver.ts +273 -0
- package/src/stitching/useIncrementalStitcher.ts +252 -0
- package/src/types.ts +78 -0
|
@@ -0,0 +1,2727 @@
|
|
|
1
|
+
// SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
//
|
|
3
|
+
// IncrementalStitcher — Swift-side engine for the live
|
|
4
|
+
// panorama-stitching pipeline introduced in
|
|
5
|
+
// docs/site-content/design/2026-04-30-realtime-incremental-stitching.md.
|
|
6
|
+
//
|
|
7
|
+
// What this file does:
|
|
8
|
+
// - Owns a single `OpenCVIncrementalStitcher` instance
|
|
9
|
+
// - Subscribes to `RNSARSession`'s per-frame ARFrame delivery
|
|
10
|
+
// - Converts ARKit pose → yaw/pitch + horizontal FoV
|
|
11
|
+
// - Dispatches addPixelBuffer onto a serial queue
|
|
12
|
+
// - Posts state updates as Notifications so the RN bridge can fan
|
|
13
|
+
// them out to JS as device events
|
|
14
|
+
//
|
|
15
|
+
// What this file deliberately does NOT do:
|
|
16
|
+
// - Touch OpenCV / cv::* — that's confined to the .mm impl behind
|
|
17
|
+
// the ObjC interface.
|
|
18
|
+
// - Manage UIKit views — the live preview is rendered via a separate
|
|
19
|
+
// ViewManager that watches the snapshot file and re-loads it.
|
|
20
|
+
// - Emit RN events directly — that's the bridge's job. This class
|
|
21
|
+
// stays free of any React.framework dependency so it can be
|
|
22
|
+
// swift-tested in isolation.
|
|
23
|
+
//
|
|
24
|
+
// Threading:
|
|
25
|
+
// ARSession delegates fire on an Apple-owned queue (~60 Hz). Inside
|
|
26
|
+
// the delegate we synchronously convert NV12 → BGR Mat (~5 ms) and
|
|
27
|
+
// compute pose-delta gating; CHEAP rejects (overlap < min, > max)
|
|
28
|
+
// short-circuit before any feature work runs. Heavy candidate
|
|
29
|
+
// processing (ORB + match + RANSAC + warp + blend) hops onto a
|
|
30
|
+
// dedicated serial queue so the AR delegate isn't blocked.
|
|
31
|
+
//
|
|
32
|
+
// Pixel-buffer lifetime:
|
|
33
|
+
// Apple guarantees ARFrame.capturedImage stays valid only within
|
|
34
|
+
// the delegate callback (see the comment on RNSARSession's
|
|
35
|
+
// recording-append path). We therefore consume the buffer
|
|
36
|
+
// inside the delegate (the .mm copies pixels into a cv::Mat — the
|
|
37
|
+
// Mat owns its own heap memory) before returning, even when the
|
|
38
|
+
// actual heavy work is dispatched to the serial queue.
|
|
39
|
+
|
|
40
|
+
import Foundation
|
|
41
|
+
import ARKit
|
|
42
|
+
import simd
|
|
43
|
+
import UIKit
|
|
44
|
+
import os.log
|
|
45
|
+
|
|
46
|
+
/// Public outcome enum mirroring the ObjC `RLISFrameOutcome` so JS
|
|
47
|
+
/// callers can inspect what happened to each frame without crossing
|
|
48
|
+
/// the ObjC++ boundary themselves.
|
|
49
|
+
///
|
|
50
|
+
/// Values 7+ are emitted from the Swift gate layer (KeyframeGate),
|
|
51
|
+
/// not from the native engine. Keep numeric values in lockstep with
|
|
52
|
+
/// `IncrementalOutcome` in incremental.ts.
|
|
53
|
+
@objc public enum IncrementalOutcome: Int {
|
|
54
|
+
case acceptedHigh = 0
|
|
55
|
+
case acceptedMedium = 1
|
|
56
|
+
case skippedTooClose = 2
|
|
57
|
+
case rejectedTooFar = 3
|
|
58
|
+
case rejectedSceneUniform = 4
|
|
59
|
+
case rejectedAlignmentLost = 5
|
|
60
|
+
case skippedTrackingPoor = 6
|
|
61
|
+
/// V12.11 — engine refused a frame because pan reversed past the
|
|
62
|
+
/// running max along the pan axis. Mirrors the JS-side enum value
|
|
63
|
+
/// that's been there since V12.11. Was missing on the Swift side
|
|
64
|
+
/// previously; the bridge fell back to `.skippedTrackingPoor` for
|
|
65
|
+
/// any rawValue >= 7.
|
|
66
|
+
case rejectedReverseDirection = 7
|
|
67
|
+
/// V16 — KeyframeGate rejected the frame: not enough new content
|
|
68
|
+
/// vs the last accepted keyframe (overlap > 1 - overlapThreshold).
|
|
69
|
+
case skippedKeyframeOverlap = 8
|
|
70
|
+
/// V16 — KeyframeGate rejected the frame: hit `keyframeMaxCount`
|
|
71
|
+
/// for the capture. Host should auto-finalize.
|
|
72
|
+
case skippedKeyframeMaxReached = 9
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/// State snapshot the bridge re-emits to JS on every accepted frame
|
|
76
|
+
/// (and on rejects when there's a hint to surface).
|
|
77
|
+
@objc(IncrementalStateObject)
|
|
78
|
+
public final class IncrementalStateObject: NSObject {
|
|
79
|
+
@objc public let panoramaPath: String?
|
|
80
|
+
@objc public let width: Int
|
|
81
|
+
@objc public let height: Int
|
|
82
|
+
@objc public let acceptedCount: Int
|
|
83
|
+
@objc public let outcome: IncrementalOutcome
|
|
84
|
+
@objc public let confidence: Double
|
|
85
|
+
@objc public let overlapPercent: Double
|
|
86
|
+
@objc public let processingMs: Double
|
|
87
|
+
/// V12.12 — engine-detected physical orientation, plumbed up
|
|
88
|
+
/// from `RLISFrameTelemetry.isLandscape`. See incremental.ts
|
|
89
|
+
/// for the full rationale (single source of truth across SDK
|
|
90
|
+
/// + host).
|
|
91
|
+
@objc public let isLandscape: Bool
|
|
92
|
+
/// V12.14.9 — running painted extent along the pan axis, in
|
|
93
|
+
/// canvas pixels. Combined with `panExtent`, lets the JS band
|
|
94
|
+
/// overlay size the thumbnail proportionally on every state
|
|
95
|
+
/// event (not just snapshot frames).
|
|
96
|
+
@objc public let paintedExtent: Int
|
|
97
|
+
/// V12.14.9 — total canvas pan-axis extent (engine config).
|
|
98
|
+
/// Constant for the lifetime of a capture. fillRatio =
|
|
99
|
+
/// `paintedExtent / panExtent`.
|
|
100
|
+
@objc public let panExtent: Int
|
|
101
|
+
/// V16 — KeyframeGate's max-keyframes cap for this capture. 0
|
|
102
|
+
/// when the gate is disabled (frameSelectionMode = "time-based"),
|
|
103
|
+
/// in which case `acceptedCount` should be displayed as a raw
|
|
104
|
+
/// counter rather than a "n / max" pill. When > 0, the JS
|
|
105
|
+
/// status pill renders "Keyframes: acceptedCount / keyframeMax".
|
|
106
|
+
@objc public let keyframeMax: Int
|
|
107
|
+
|
|
108
|
+
@objc public init(
|
|
109
|
+
panoramaPath: String?,
|
|
110
|
+
width: Int,
|
|
111
|
+
height: Int,
|
|
112
|
+
acceptedCount: Int,
|
|
113
|
+
outcome: IncrementalOutcome,
|
|
114
|
+
confidence: Double,
|
|
115
|
+
overlapPercent: Double,
|
|
116
|
+
processingMs: Double,
|
|
117
|
+
isLandscape: Bool,
|
|
118
|
+
paintedExtent: Int,
|
|
119
|
+
panExtent: Int,
|
|
120
|
+
keyframeMax: Int = 0
|
|
121
|
+
) {
|
|
122
|
+
self.panoramaPath = panoramaPath
|
|
123
|
+
self.width = width
|
|
124
|
+
self.height = height
|
|
125
|
+
self.acceptedCount = acceptedCount
|
|
126
|
+
self.outcome = outcome
|
|
127
|
+
self.confidence = confidence
|
|
128
|
+
self.overlapPercent = overlapPercent
|
|
129
|
+
self.processingMs = processingMs
|
|
130
|
+
self.isLandscape = isLandscape
|
|
131
|
+
self.paintedExtent = paintedExtent
|
|
132
|
+
self.panExtent = panExtent
|
|
133
|
+
self.keyframeMax = keyframeMax
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
@objc public func asDictionary() -> [String: Any] {
|
|
137
|
+
var dict: [String: Any] = [
|
|
138
|
+
"width": width,
|
|
139
|
+
"height": height,
|
|
140
|
+
"acceptedCount": acceptedCount,
|
|
141
|
+
"outcome": outcome.rawValue,
|
|
142
|
+
"confidence": confidence,
|
|
143
|
+
"overlapPercent": overlapPercent,
|
|
144
|
+
"processingMs": processingMs,
|
|
145
|
+
"isLandscape": isLandscape,
|
|
146
|
+
"paintedExtent": paintedExtent,
|
|
147
|
+
"panExtent": panExtent,
|
|
148
|
+
"keyframeMax": keyframeMax,
|
|
149
|
+
]
|
|
150
|
+
if let p = panoramaPath { dict["panoramaPath"] = p }
|
|
151
|
+
return dict
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
/// Notification names — bridge subscribes to these and re-emits as
|
|
156
|
+
/// React Native device events. Keeping the engine framework-free
|
|
157
|
+
/// keeps Swift unit tests viable.
|
|
158
|
+
public extension Notification.Name {
|
|
159
|
+
static let retailensIncrementalStateUpdate =
|
|
160
|
+
Notification.Name("IncrementalStateUpdate")
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
// MARK: - FinalizePayload (C2 — stateless finalize, fix-attempt 8)
|
|
165
|
+
//
|
|
166
|
+
// Value-typed snapshot of every input the finalize closures need.
|
|
167
|
+
// Built in `finalize()`'s prologue under `stateLock` and passed BY
|
|
168
|
+
// VALUE into the workQueue closure so the closure can capture
|
|
169
|
+
// `[payload, completion]` ONLY — zero `self` references — closing
|
|
170
|
+
// the entire class of `objc_retain`-on-torn-pointer crashes that
|
|
171
|
+
// fix-attempts 1-7 chased symptom-by-symptom on the same code path.
|
|
172
|
+
//
|
|
173
|
+
// Per design doc 2026-05-12-finalize-crash-investigation.md (C2
|
|
174
|
+
// escalation): all prior fixes attacked specific symptoms (which
|
|
175
|
+
// ivar's read raced, which lock's discipline was lax). This is the
|
|
176
|
+
// architectural escalation: by construction the closure has no
|
|
177
|
+
// access to mutable stitcher state, so no race in this code path is
|
|
178
|
+
// even possible.
|
|
179
|
+
//
|
|
180
|
+
// File-scope `internal` (not private) so future ports (Design 2
|
|
181
|
+
// actor, Design 3 C++) can adopt the same payload as their input.
|
|
182
|
+
//
|
|
183
|
+
// MAINTENANCE INVARIANT: every ivar finalize closures currently
|
|
184
|
+
// read (or might read in future edits) MUST live here. If you add
|
|
185
|
+
// a new finalize-relevant ivar to IncrementalStitcher,
|
|
186
|
+
// thread it through this struct. The CI test at
|
|
187
|
+
// scripts/check_c2_invariant.sh prevents accidental `self.*`
|
|
188
|
+
// reintroduction inside the closure body.
|
|
189
|
+
struct FinalizePayload {
|
|
190
|
+
// ── Output destination + quality ─────────────────────────────
|
|
191
|
+
/// The caller-supplied panorama output path, normalized
|
|
192
|
+
/// (file:// stripped if present).
|
|
193
|
+
let cleaned: String
|
|
194
|
+
/// JPEG quality, clamped to 1...100.
|
|
195
|
+
let q: Int
|
|
196
|
+
|
|
197
|
+
// ── Stitcher mode selection ─────────────────────────────────
|
|
198
|
+
/// True if this finalize is the V16 batch-keyframe pipeline.
|
|
199
|
+
let inBatchKeyframeMode: Bool
|
|
200
|
+
/// Hybrid engine ref (V14/V15 path). nil if batch mode.
|
|
201
|
+
let hybrid: OpenCVIncrementalStitcher?
|
|
202
|
+
/// First-wins cylindrical engine ref (V13 path). nil if batch mode.
|
|
203
|
+
let slit: OpenCVFirstWinsCylindricalStitcher?
|
|
204
|
+
/// V16 keyframe collector — owns the per-session JPEG sidecar
|
|
205
|
+
/// directory the post-stitch result references.
|
|
206
|
+
let collector: OpenCVKeyframeCollector?
|
|
207
|
+
|
|
208
|
+
// ── Frame inputs ─────────────────────────────────────────────
|
|
209
|
+
/// Absolute paths to keyframe JPEGs. Value-copied; the
|
|
210
|
+
/// underlying String storage is COW and immutable.
|
|
211
|
+
let paths: [String]
|
|
212
|
+
|
|
213
|
+
// ── cv::Stitcher tuning snapshots (fix4 lineage) ─────────────
|
|
214
|
+
let batchWarperType: String
|
|
215
|
+
let batchBlenderType: String
|
|
216
|
+
let batchSeamFinderType: String
|
|
217
|
+
let batchEnableInscribedRectCrop: Bool
|
|
218
|
+
let keyframeExifOrientation: Int
|
|
219
|
+
/// AR-STITCHING-TWO-MODES (memory/ar-stitching-two-modes.md):
|
|
220
|
+
/// capture-time hold orientation for the bake-rotation pass.
|
|
221
|
+
let captureOrientation: String
|
|
222
|
+
|
|
223
|
+
// ── Result metadata ──────────────────────────────────────────
|
|
224
|
+
/// Backpressure drops accumulated this capture, surfaced to JS.
|
|
225
|
+
let drops: Int
|
|
226
|
+
/// Whether the AR session was running at finalize-entry — drives
|
|
227
|
+
/// the AR restart in the closure's defer.
|
|
228
|
+
let arWasRunning: Bool
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
@objc(IncrementalStitcher)
|
|
233
|
+
public final class IncrementalStitcher: NSObject {
|
|
234
|
+
|
|
235
|
+
/// V13.0c.1.1 — same os_log subsystem as the slit-scan engine's
|
|
236
|
+
/// SlitDiagLog so Console.app sees both V13.0b-gate and V13.0c-trans
|
|
237
|
+
/// under one filter. FAULT-level survives NSLog's burst rate-limit
|
|
238
|
+
/// (~10/sec) — diagnostic logs at 50fps would otherwise be dropped.
|
|
239
|
+
fileprivate static let diagLog = OSLog(
|
|
240
|
+
subsystem: "com.tiger.retailens.sdk",
|
|
241
|
+
category: "slitscan"
|
|
242
|
+
)
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
@objc public static let shared = IncrementalStitcher()
|
|
246
|
+
|
|
247
|
+
/// Underlying OpenCV engine. Created on `start`, torn down on
|
|
248
|
+
/// `finalize`/`reset`. Holding it across captures would keep the
|
|
249
|
+
/// 24 MB canvas allocated in idle.
|
|
250
|
+
///
|
|
251
|
+
/// V10: two engine variants exist behind one Swift wrapper.
|
|
252
|
+
/// `hybridEngine` (Samsung-style, full-frame cylindrical + OF) is
|
|
253
|
+
/// the default. `firstwinsEngine` (Apple-style, per-strip painting)
|
|
254
|
+
/// is opt-in via the JS `engine: 'slitscan'` start option. Only
|
|
255
|
+
/// one is non-nil at a time.
|
|
256
|
+
private var hybridEngine: OpenCVIncrementalStitcher?
|
|
257
|
+
private var firstwinsEngine: OpenCVFirstWinsCylindricalStitcher?
|
|
258
|
+
|
|
259
|
+
/// V15.0b — true once we've forwarded the latched plane transform
|
|
260
|
+
/// from RNSARSession to the slit-scan engine. Reset on
|
|
261
|
+
/// every start() so the next capture re-propagates. We only
|
|
262
|
+
/// forward once per capture: the plane transform is latched
|
|
263
|
+
/// (RNSARSession ignores subsequent ARKit refinements),
|
|
264
|
+
/// so re-propagating each frame is wasted work.
|
|
265
|
+
private var havePropagatedPlane: Bool = false
|
|
266
|
+
|
|
267
|
+
/// Convenience: read the active engine's accepted count. Used by
|
|
268
|
+
/// the per-frame state event.
|
|
269
|
+
private var engineAcceptedCount: Int {
|
|
270
|
+
return hybridEngine?.acceptedCount ?? firstwinsEngine?.acceptedCount ?? 0
|
|
271
|
+
}
|
|
272
|
+
private var anyEngineActive: Bool {
|
|
273
|
+
return hybridEngine != nil || firstwinsEngine != nil
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
/// Serial queue for the heavy per-frame work. ARSession delegate
|
|
277
|
+
/// only dispatches a pre-allocated cv::Mat onto this queue — the
|
|
278
|
+
/// pixel buffer itself is consumed before return.
|
|
279
|
+
///
|
|
280
|
+
/// 2026-05-15 (C3 deferral) — investigated splitting the batch-
|
|
281
|
+
/// keyframe stitch onto its own DispatchQueue so a slow stitch
|
|
282
|
+
/// doesn't block ARSession frame ingestion on this workQueue.
|
|
283
|
+
/// Backed out: the existing `workQueue.sync` finalize boundary
|
|
284
|
+
/// (V16 Phase 1b.fix6) is INTENTIONAL — it serialises finalize
|
|
285
|
+
/// against in-flight frame work to prevent state-event races.
|
|
286
|
+
/// Moving the stitch to a separate queue requires reworking the
|
|
287
|
+
/// completion handler + stateLock contract to provide the same
|
|
288
|
+
/// ordering guarantees the sync barrier currently gives. Real
|
|
289
|
+
/// fix is non-trivial; deferred until pose-driven stitch work
|
|
290
|
+
/// lands (which will rework the queue topology anyway).
|
|
291
|
+
private let workQueue = DispatchQueue(
|
|
292
|
+
label: "com.retailens.incremental.stitcher",
|
|
293
|
+
qos: .userInitiated
|
|
294
|
+
)
|
|
295
|
+
|
|
296
|
+
/// 2026-05-16 — realtime+batch fusion (Option A "Replace on
|
|
297
|
+
/// completion"). Dedicated queue for the async refinement run
|
|
298
|
+
/// that follows a hybrid-engine finalize(). Kept SEPARATE from
|
|
299
|
+
/// `workQueue` so the next capture's start/consumeFrame path
|
|
300
|
+
/// isn't gated on the prior capture's 2-5 s cv::Stitcher run —
|
|
301
|
+
/// the design doc explicitly calls out "operator can continue
|
|
302
|
+
/// browsing / starting another capture during refinement".
|
|
303
|
+
///
|
|
304
|
+
/// Serial: at most one refinement runs at a time (the design's
|
|
305
|
+
/// "cancellation semantics if a new capture starts mid-refine"
|
|
306
|
+
/// is out of scope for this MVP — see prompt's "deliberately out
|
|
307
|
+
/// of scope" list).
|
|
308
|
+
private let refineQueue = DispatchQueue(
|
|
309
|
+
label: "com.retailens.incremental.refine",
|
|
310
|
+
qos: .utility
|
|
311
|
+
)
|
|
312
|
+
|
|
313
|
+
/// Lock guarding `engine`/`isRunning` reads/writes. ARSession
|
|
314
|
+
/// delegate uses `try` to avoid blocking ARKit; if start/stop is
|
|
315
|
+
/// mid-flight the frame is dropped.
|
|
316
|
+
private let stateLock = NSLock()
|
|
317
|
+
|
|
318
|
+
/// Whether the engine is currently active. Set by start/stop.
|
|
319
|
+
@objc public private(set) var isRunning: Bool = false
|
|
320
|
+
|
|
321
|
+
/// The most recent state snapshot — readable by JS via the
|
|
322
|
+
/// bridge's `getState`.
|
|
323
|
+
private var lastState: IncrementalStateObject?
|
|
324
|
+
|
|
325
|
+
/// Cumulative drop counter — frames the work queue couldn't keep
|
|
326
|
+
/// up with. Diagnostic only; not surfaced to JS.
|
|
327
|
+
private var droppedBackpressure: Int = 0
|
|
328
|
+
|
|
329
|
+
/// V11 Gap #27: true when an ingest is in flight on the work
|
|
330
|
+
/// queue. Subsequent AR delegate frames are dropped (rather
|
|
331
|
+
/// than queued) so latency between AR time and canvas state
|
|
332
|
+
/// stays bounded.
|
|
333
|
+
private var workInFlight: Bool = false
|
|
334
|
+
|
|
335
|
+
/// Snapshot quality — host can pass on start.
|
|
336
|
+
private var snapshotJpegQuality: Int = 75
|
|
337
|
+
|
|
338
|
+
/// Periodic snapshot cadence — emit a panoramaPath update at most
|
|
339
|
+
/// every N accepts. Default 1 (every accept) gives the freshest
|
|
340
|
+
/// preview; field testing may show batching is friendlier on
|
|
341
|
+
/// the JS image-cache.
|
|
342
|
+
private var snapshotEveryNAccepts: Int = 1
|
|
343
|
+
|
|
344
|
+
/// Internal counter used with snapshotEveryNAccepts.
|
|
345
|
+
private var acceptsSinceSnapshot: Int = 0
|
|
346
|
+
|
|
347
|
+
/// V13.0c.1 — diagnostic state: first-frame world translation.
|
|
348
|
+
/// Used to compute Δtranslation per frame for the
|
|
349
|
+
/// [V13.0c-trans] log. We need to know how much users actually
|
|
350
|
+
/// translate during typical captures before committing to the
|
|
351
|
+
/// per-pixel depth correction architecture (V13.0c.2-.4).
|
|
352
|
+
/// Reset on each new capture (handled in `start()` below).
|
|
353
|
+
private var firstFrameTx: Double = 0
|
|
354
|
+
private var firstFrameTy: Double = 0
|
|
355
|
+
private var firstFrameTz: Double = 0
|
|
356
|
+
private var hasFirstFrameTranslation: Bool = false
|
|
357
|
+
private var consumeFrameCounter: Int = 0
|
|
358
|
+
|
|
359
|
+
/// V16 — pose-driven keyframe gate. When `enabled` (set from the
|
|
360
|
+
/// JS `frameSelectionMode = "pose-based"` config), each ARFrame is
|
|
361
|
+
/// projected onto the latched ARKit plane and accepted only when
|
|
362
|
+
/// it has ≥ `overlapThreshold` of NEW content vs the last
|
|
363
|
+
/// accepted keyframe. Bounded to `maxCount` keyframes per
|
|
364
|
+
/// capture. See KeyframeGate.swift for the full rationale.
|
|
365
|
+
private let keyframeGate = KeyframeGate()
|
|
366
|
+
|
|
367
|
+
/// V16 Phase 1 — when `engineMode == "batch-keyframe"`, no
|
|
368
|
+
/// incremental engine runs; we accumulate the gate-accepted
|
|
369
|
+
/// frames as on-disk JPEGs + their poses, then on `finalize` hand
|
|
370
|
+
/// them to `OpenCVStitcher.stitchKeyframePaths:withPoses:` (the
|
|
371
|
+
/// full BA + ExposureCompensator + GraphCutSeamFinder +
|
|
372
|
+
/// MultiBandBlender pipeline) for one-shot stitching. Why this
|
|
373
|
+
/// is structurally different from the slit-scan / hybrid engines:
|
|
374
|
+
/// they ingest into a streaming canvas, whereas batch-keyframe
|
|
375
|
+
/// defers all stitching until shutter release so the global-
|
|
376
|
+
/// stage quality wins (BA, multi-band) become available.
|
|
377
|
+
private var batchKeyframeMode: Bool = false
|
|
378
|
+
private var keyframeCollector: OpenCVKeyframeCollector?
|
|
379
|
+
/// Poses recorded 1:1 with `keyframeCollector`'s saved JPEGs.
|
|
380
|
+
/// Each entry is `RNSARFramePose.asDictionary()`. Reset
|
|
381
|
+
/// on every `start()`.
|
|
382
|
+
private var keyframePoses: [[String: Any]] = []
|
|
383
|
+
/// Saved JPEG paths in capture order. Tracked separately from
|
|
384
|
+
/// the collector so finalize doesn't have to reach back into ObjC.
|
|
385
|
+
private var keyframePaths: [String] = []
|
|
386
|
+
/// Frame rotation degrees passed at `start()` — needed when
|
|
387
|
+
/// saving keyframes so the JPEGs land in user-pan orientation
|
|
388
|
+
/// (the stitcher reads them in that orientation).
|
|
389
|
+
private var keyframeRotationDegrees: Int = 90
|
|
390
|
+
/// V16 Phase 1.fix2 — EXIF Orientation tag (1..8) baked into
|
|
391
|
+
/// each saved keyframe JPEG so iOS image renderers display
|
|
392
|
+
/// correctly while the stitcher (with IMREAD_IGNORE_ORIENTATION)
|
|
393
|
+
/// gets raw landscape pixels matching the pose's intrinsics.
|
|
394
|
+
private var keyframeExifOrientation: Int = 1
|
|
395
|
+
/// V16 Phase 1.fix4 — cv::Stitcher knobs. Defaults match the
|
|
396
|
+
/// modal's defaults and the legacy batch path defaults
|
|
397
|
+
/// (PlaneWarper + MultiBandBlender + GraphCutSeamFinder) since
|
|
398
|
+
/// fix4 routes through cv::Stitcher's feature-matched pipeline
|
|
399
|
+
/// where these defaults are the production-tested combo. User
|
|
400
|
+
/// can override via the modal Projection / Blender / Seam-finder
|
|
401
|
+
/// sections — those values flow through configOverrides at
|
|
402
|
+
/// start().
|
|
403
|
+
private var batchWarperType: String = "plane"
|
|
404
|
+
private var batchBlenderType: String = "multiband"
|
|
405
|
+
private var batchSeamFinderType: String = "graphcut"
|
|
406
|
+
// V16 Phase 1b.fix5c — operator-visible toggle for the
|
|
407
|
+
// max-inscribed-rectangle crop in the batch-keyframe finalize.
|
|
408
|
+
// Default OFF. When OFF, native crops to bbox only; when ON,
|
|
409
|
+
// the inscribed-rect + morph-close + col-projection pipeline
|
|
410
|
+
// runs.
|
|
411
|
+
private var batchEnableInscribedRectCrop: Bool = false
|
|
412
|
+
/// AR-STITCHING-TWO-MODES — see memory/ar-stitching-two-modes.md
|
|
413
|
+
///
|
|
414
|
+
/// Physical phone orientation at start() time, sourced from the
|
|
415
|
+
/// JS accelerometer hook. Used to drive the OUTPUT panorama's
|
|
416
|
+
/// bake-rotation in OpenCVStitcher.stitchFramePaths. Held for
|
|
417
|
+
/// the lifetime of the capture so finalize() doesn't need to
|
|
418
|
+
/// re-sample (the user may have rotated the phone mid-pan and
|
|
419
|
+
/// we want the rotation that was correct WHEN they started).
|
|
420
|
+
///
|
|
421
|
+
/// Valid values mirror IncrementalStartOptions.captureOrientation
|
|
422
|
+
/// in the JS API: "portrait", "portrait-upside-down",
|
|
423
|
+
/// "landscape-left", "landscape-right". Any other string is
|
|
424
|
+
/// treated as "portrait" (no rotation) by the .mm side.
|
|
425
|
+
private var captureOrientation: String = "portrait"
|
|
426
|
+
|
|
427
|
+
private override init() {
|
|
428
|
+
super.init()
|
|
429
|
+
}
|
|
430
|
+
|
|
431
|
+
/// 2026-05-18 (iOS cross-orientation fix) — bridge entry-point
|
|
432
|
+
/// that the bridge calls in finalize() when JS supplies a fresh
|
|
433
|
+
/// orientation. Overrides whatever start() snapshotted
|
|
434
|
+
/// (native ARKit query OR JS fallback). Used to close the bug
|
|
435
|
+
/// where the user opens the screen in orientation A, captures
|
|
436
|
+
/// in orientation B, and the bake_rotation table runs against
|
|
437
|
+
/// orientation A (the start-time value).
|
|
438
|
+
///
|
|
439
|
+
/// Caller responsibility: this should only be called BEFORE
|
|
440
|
+
/// finalize() begins the stitch. Calling concurrently with an
|
|
441
|
+
/// in-flight stitch is a race on `self.captureOrientation`
|
|
442
|
+
/// (which the stitcher reads through the payload snapshot at
|
|
443
|
+
/// finalize() prologue). The bridge enforces "update then
|
|
444
|
+
/// finalize" sequentially on the workQueue.
|
|
445
|
+
@objc public func updateCaptureOrientation(_ orientation: String) {
|
|
446
|
+
stateLock.lock()
|
|
447
|
+
let prev = self.captureOrientation
|
|
448
|
+
self.captureOrientation = orientation
|
|
449
|
+
stateLock.unlock()
|
|
450
|
+
os_log(.fault, log: Self.diagLog,
|
|
451
|
+
"[V16-orchestrator] updateCaptureOrientation: %{public}@ → %{public}@",
|
|
452
|
+
prev, orientation)
|
|
453
|
+
}
|
|
454
|
+
|
|
455
|
+
/// 2026-05-18 (Iss 3) — return the current capture's keyframe
|
|
456
|
+
/// session directory, or nil if no capture is in flight / engine
|
|
457
|
+
/// isn't using a per-session keyframe collector.
|
|
458
|
+
@objc public func currentKeyframeDir() -> String? {
|
|
459
|
+
stateLock.lock()
|
|
460
|
+
defer { stateLock.unlock() }
|
|
461
|
+
guard self.isRunning, self.batchKeyframeMode else { return nil }
|
|
462
|
+
return self.keyframeCollector?.sessionDir
|
|
463
|
+
}
|
|
464
|
+
|
|
465
|
+
/// 2026-05-18 (Iss 3) — GC stale keyframe-session directories.
|
|
466
|
+
///
|
|
467
|
+
/// Scans `Library/Application Support/Captures/` for subdirectories
|
|
468
|
+
/// whose newest file mtime is older than `cutoffMs` (ms past epoch).
|
|
469
|
+
/// Each stale subdirectory is removed in full (it's a session UUID
|
|
470
|
+
/// dir with N keyframe JPEGs). Sessions whose newest file is newer
|
|
471
|
+
/// than the cutoff are LEFT ALONE — even if they look stale by some
|
|
472
|
+
/// other heuristic — because they might belong to a capture that's
|
|
473
|
+
/// still in flight (the engine writes incremental frames to the
|
|
474
|
+
/// same dir).
|
|
475
|
+
///
|
|
476
|
+
/// Returns a tuple of (sessionsDeleted, bytesFreed) so the bridge
|
|
477
|
+
/// can surface those numbers to JS for an optional UX toast. All
|
|
478
|
+
/// filesystem errors are swallowed and counted as "not deleted";
|
|
479
|
+
/// host should NEVER see a thrown error from this — at worst it
|
|
480
|
+
/// gets back zero counts and can investigate Console.app logs.
|
|
481
|
+
@objc public func cleanupKeyframes(
|
|
482
|
+
olderThanMs: Double
|
|
483
|
+
) -> [String: NSNumber] {
|
|
484
|
+
let cutoff = Date().timeIntervalSince1970 - (olderThanMs / 1000.0)
|
|
485
|
+
let fm = FileManager.default
|
|
486
|
+
guard let appSupport = try? fm.url(
|
|
487
|
+
for: .applicationSupportDirectory,
|
|
488
|
+
in: .userDomainMask,
|
|
489
|
+
appropriateFor: nil,
|
|
490
|
+
create: false // don't create if missing — that means nothing to clean
|
|
491
|
+
) else {
|
|
492
|
+
return ["sessionsDeleted": 0, "bytesFreed": 0]
|
|
493
|
+
}
|
|
494
|
+
let capturesURL = appSupport.appendingPathComponent("Captures", isDirectory: true)
|
|
495
|
+
guard fm.fileExists(atPath: capturesURL.path) else {
|
|
496
|
+
return ["sessionsDeleted": 0, "bytesFreed": 0]
|
|
497
|
+
}
|
|
498
|
+
guard let sessions = try? fm.contentsOfDirectory(
|
|
499
|
+
at: capturesURL,
|
|
500
|
+
includingPropertiesForKeys: [.contentModificationDateKey, .isDirectoryKey, .fileSizeKey],
|
|
501
|
+
options: [.skipsHiddenFiles]
|
|
502
|
+
) else {
|
|
503
|
+
return ["sessionsDeleted": 0, "bytesFreed": 0]
|
|
504
|
+
}
|
|
505
|
+
|
|
506
|
+
var sessionsDeleted = 0
|
|
507
|
+
var bytesFreed: UInt64 = 0
|
|
508
|
+
for sessionURL in sessions {
|
|
509
|
+
// Only dirs are real sessions; skip stray files.
|
|
510
|
+
let isDir = (try? sessionURL.resourceValues(
|
|
511
|
+
forKeys: [.isDirectoryKey]
|
|
512
|
+
))?.isDirectory ?? false
|
|
513
|
+
if !isDir { continue }
|
|
514
|
+
// Newest mtime across the session's files (recursive — though
|
|
515
|
+
// the collector writes a flat dir today, future-proof).
|
|
516
|
+
var newestMtime: TimeInterval = 0
|
|
517
|
+
var sessionBytes: UInt64 = 0
|
|
518
|
+
if let enumerator = fm.enumerator(
|
|
519
|
+
at: sessionURL,
|
|
520
|
+
includingPropertiesForKeys: [.contentModificationDateKey, .fileSizeKey],
|
|
521
|
+
options: [.skipsHiddenFiles]
|
|
522
|
+
) {
|
|
523
|
+
for case let fileURL as URL in enumerator {
|
|
524
|
+
let r = try? fileURL.resourceValues(
|
|
525
|
+
forKeys: [.contentModificationDateKey, .fileSizeKey]
|
|
526
|
+
)
|
|
527
|
+
if let mtime = r?.contentModificationDate?.timeIntervalSince1970 {
|
|
528
|
+
if mtime > newestMtime { newestMtime = mtime }
|
|
529
|
+
}
|
|
530
|
+
if let bytes = r?.fileSize {
|
|
531
|
+
sessionBytes += UInt64(bytes)
|
|
532
|
+
}
|
|
533
|
+
}
|
|
534
|
+
}
|
|
535
|
+
// Use the directory's own mtime as a fallback if no files
|
|
536
|
+
// matched (an empty session dir is also stale).
|
|
537
|
+
if newestMtime == 0 {
|
|
538
|
+
if let dirMtime = (try? sessionURL.resourceValues(
|
|
539
|
+
forKeys: [.contentModificationDateKey]
|
|
540
|
+
))?.contentModificationDate?.timeIntervalSince1970 {
|
|
541
|
+
newestMtime = dirMtime
|
|
542
|
+
}
|
|
543
|
+
}
|
|
544
|
+
if newestMtime > 0 && newestMtime < cutoff {
|
|
545
|
+
if (try? fm.removeItem(at: sessionURL)) != nil {
|
|
546
|
+
sessionsDeleted += 1
|
|
547
|
+
bytesFreed += sessionBytes
|
|
548
|
+
}
|
|
549
|
+
}
|
|
550
|
+
}
|
|
551
|
+
os_log(.fault, log: Self.diagLog,
|
|
552
|
+
"[V16-orchestrator] cleanupKeyframes olderThanMs=%.0f sessions=%d bytes=%llu",
|
|
553
|
+
olderThanMs, Int32(sessionsDeleted), bytesFreed)
|
|
554
|
+
return [
|
|
555
|
+
"sessionsDeleted": NSNumber(value: sessionsDeleted),
|
|
556
|
+
"bytesFreed": NSNumber(value: bytesFreed),
|
|
557
|
+
]
|
|
558
|
+
}
|
|
559
|
+
|
|
560
|
+
/// 2026-05-16 — realtime+batch fusion (Option A) path derivation.
|
|
561
|
+
/// Given the live panorama path (which finalize() wrote inside
|
|
562
|
+
/// the app sandbox tmp or a host-supplied location), pick a path
|
|
563
|
+
/// for the refined output. Pattern:
|
|
564
|
+
///
|
|
565
|
+
/// /…/RNImageStitcherIncremental-<uuid>.jpg
|
|
566
|
+
/// → /…/RNImageStitcherIncremental-<uuid>-refined.jpg
|
|
567
|
+
///
|
|
568
|
+
/// Same directory keeps cleanup discoverable (delete both when
|
|
569
|
+
/// the audit is discarded). Different name avoids racing the
|
|
570
|
+
/// host UI that may still be reading the live file as the
|
|
571
|
+
/// refinement is writing.
|
|
572
|
+
fileprivate static func refinedPathFromLive(livePath: String) -> String {
|
|
573
|
+
let ns = livePath as NSString
|
|
574
|
+
let dir = ns.deletingLastPathComponent
|
|
575
|
+
let base = (ns.lastPathComponent as NSString).deletingPathExtension
|
|
576
|
+
let ext = (ns.lastPathComponent as NSString).pathExtension
|
|
577
|
+
let refinedName = ext.isEmpty
|
|
578
|
+
? "\(base)-refined"
|
|
579
|
+
: "\(base)-refined.\(ext)"
|
|
580
|
+
return (dir as NSString).appendingPathComponent(refinedName)
|
|
581
|
+
}
|
|
582
|
+
|
|
583
|
+
// ── Native orientation classifier ────────────────────────────────
|
|
584
|
+
//
|
|
585
|
+
// AR-STITCHING-TWO-MODES — see memory/ar-stitching-two-modes.md
|
|
586
|
+
//
|
|
587
|
+
// Why this exists: JS's useDeviceOrientation hook ships React
|
|
588
|
+
// state that's stale at the moment incremental.start() is invoked
|
|
589
|
+
// (the hook samples accelerometer at 10 Hz and renders through
|
|
590
|
+
// React's normal update cycle — by the time the start() callback
|
|
591
|
+
// runs, the closure has captured the previous-state value).
|
|
592
|
+
// Field test on 2026-05-11 confirmed: 3/3 captures (Mode B,
|
|
593
|
+
// Mode A landscape-left, Mode A landscape-right) all arrived at
|
|
594
|
+
// the bridge with captureOrientation="portrait" even though the
|
|
595
|
+
// user explicitly rotated to landscape for two of them.
|
|
596
|
+
//
|
|
597
|
+
// ARKit gives us the answer directly: `frame.camera.eulerAngles.z`
|
|
598
|
+
// is the camera's roll (rotation around its optical axis) — the
|
|
599
|
+
// exact value that distinguishes portrait / landscape-left /
|
|
600
|
+
// landscape-right / portrait-upside-down regardless of any UI
|
|
601
|
+
// lock or JS state staleness. Read it synchronously at start()
|
|
602
|
+
// time and use it as the source of truth.
|
|
603
|
+
//
|
|
604
|
+
// Convention (verified against ARKit docs + iOS device axes):
|
|
605
|
+
// • Camera local axes: +X right, +Y up (along phone length
|
|
606
|
+
// when held portrait), +Z back (toward user's face).
|
|
607
|
+
// • Roll = rotation around camera's +Z (right-hand rule).
|
|
608
|
+
// • Portrait → roll ≈ 0
|
|
609
|
+
// • Landscape-left (CCW) → roll ≈ +π/2 (verified empirically;
|
|
610
|
+
// swap with -right if first test
|
|
611
|
+
// shows landscape-right hitting
|
|
612
|
+
// this branch instead)
|
|
613
|
+
// • Portrait-upside-down → roll ≈ ±π
|
|
614
|
+
// • Landscape-right (CW) → roll ≈ -π/2
|
|
615
|
+
//
|
|
616
|
+
// ±45° tolerance per quadrant keeps classification stable under
|
|
617
|
+
// small hand wobble.
|
|
618
|
+
private static func nativeCaptureOrientation() -> (
|
|
619
|
+
orientation: String,
|
|
620
|
+
rollDegrees: Double,
|
|
621
|
+
hadFrame: Bool
|
|
622
|
+
) {
|
|
623
|
+
guard let frame = RNSARSession.shared.arSession.currentFrame else {
|
|
624
|
+
// No AR frame yet — falls back to "portrait" (Mode B start
|
|
625
|
+
// state). Should be rare: incremental.start() requires
|
|
626
|
+
// ARSession to be running, which means frames are flowing.
|
|
627
|
+
return ("portrait", 0.0, false)
|
|
628
|
+
}
|
|
629
|
+
let rollRadians = Double(frame.camera.eulerAngles.z)
|
|
630
|
+
let rollDegrees = rollRadians * 180.0 / .pi
|
|
631
|
+
let classified: String
|
|
632
|
+
// Empirically calibrated against Ram's 2026-05-11 3-capture
|
|
633
|
+
// test (1st=L-left, 2nd=portrait, 3rd=L-right):
|
|
634
|
+
// Ram's "landscape-left" → roll ≈ 0° (NOT -90° as I assumed)
|
|
635
|
+
// Ram's "portrait" → roll ≈ -90°
|
|
636
|
+
// Ram's "landscape-right" → roll ≈ ±180°
|
|
637
|
+
// Ram's "portrait-upside-down" → roll ≈ +90° (by symmetry, untested)
|
|
638
|
+
//
|
|
639
|
+
// Why this differs from the device-orientation intuition:
|
|
640
|
+
// ARKit's `camera.eulerAngles.z` is the camera's roll around
|
|
641
|
+
// its optical axis, measured against world-up. The iPhone's
|
|
642
|
+
// image sensor is mounted such that its long axis aligns
|
|
643
|
+
// with the phone's WIDTH (not length), so the camera's +Y
|
|
644
|
+
// is naturally aligned with world-UP when the phone is in
|
|
645
|
+
// landscape — roll = 0 means the phone is landscape.
|
|
646
|
+
// Holding the phone in portrait rotates the camera +Y to
|
|
647
|
+
// horizontal in world → roll = ±90°.
|
|
648
|
+
//
|
|
649
|
+
// The rotation table in OpenCVStitcher.mm (landscape-left →
|
|
650
|
+
// 90° CCW, portrait → none, etc.) is unchanged — only the
|
|
651
|
+
// classifier mapping is shifted by 90°.
|
|
652
|
+
if abs(rollDegrees) < 45 {
|
|
653
|
+
classified = "landscape-left"
|
|
654
|
+
} else if rollDegrees >= 45 && rollDegrees < 135 {
|
|
655
|
+
classified = "portrait-upside-down"
|
|
656
|
+
} else if rollDegrees <= -45 && rollDegrees > -135 {
|
|
657
|
+
classified = "portrait"
|
|
658
|
+
} else {
|
|
659
|
+
classified = "landscape-right"
|
|
660
|
+
}
|
|
661
|
+
return (classified, rollDegrees, true)
|
|
662
|
+
}
|
|
663
|
+
|
|
664
|
+
// ── Public lifecycle ────────────────────────────────────────────
|
|
665
|
+
|
|
666
|
+
/// Begin a new incremental capture. Hooks the ARSession's
|
|
667
|
+
/// per-frame stream into the engine. Caller must already have
|
|
668
|
+
/// the AR session running (start/stop is the host app's job).
|
|
669
|
+
@objc public func start(
|
|
670
|
+
composeWidth: Int,
|
|
671
|
+
composeHeight: Int,
|
|
672
|
+
canvasWidth: Int,
|
|
673
|
+
canvasHeight: Int,
|
|
674
|
+
featherPx: Int,
|
|
675
|
+
snapshotJpegQuality: Int,
|
|
676
|
+
snapshotEveryNAccepts: Int,
|
|
677
|
+
frameRotationDegrees: Int,
|
|
678
|
+
engineMode: String,
|
|
679
|
+
captureOrientation: String = "portrait",
|
|
680
|
+
configOverrides: [String: Any] = [:],
|
|
681
|
+
// 2026-05-18 (Issue #2 regression fix): "arSession" (default,
|
|
682
|
+
// legacy) registers as the ARSession's frame consumer.
|
|
683
|
+
// "jsDriver" skips that registration — frames will come in
|
|
684
|
+
// via processFrameAtPath instead. Used by iOS non-AR
|
|
685
|
+
// captures (the vision-camera + gyro driver path).
|
|
686
|
+
frameSourceMode: String = "arSession"
|
|
687
|
+
) {
|
|
688
|
+
stateLock.lock()
|
|
689
|
+
if isRunning {
|
|
690
|
+
stateLock.unlock()
|
|
691
|
+
return
|
|
692
|
+
}
|
|
693
|
+
// AR-STITCHING-TWO-MODES — see memory/ar-stitching-two-modes.md
|
|
694
|
+
// Override the JS-supplied captureOrientation with a native
|
|
695
|
+
// ARKit reading. Field test on 2026-05-11 confirmed JS state
|
|
696
|
+
// is consistently stale at start() time (always "portrait"
|
|
697
|
+
// regardless of physical orientation); native AR frame data
|
|
698
|
+
// is real-time and unambiguous. See nativeCaptureOrientation
|
|
699
|
+
// for the full RCA. JS-supplied value retained in the log
|
|
700
|
+
// for diagnostic purposes.
|
|
701
|
+
let nativeResult = Self.nativeCaptureOrientation()
|
|
702
|
+
let resolvedOrientation = nativeResult.hadFrame
|
|
703
|
+
? nativeResult.orientation
|
|
704
|
+
: captureOrientation // no AR frame yet — fall back to JS
|
|
705
|
+
self.captureOrientation = resolvedOrientation
|
|
706
|
+
os_log(.fault, log: Self.diagLog,
|
|
707
|
+
"[V16-orchestrator] start: JS sent=%{public}@ native roll=%.1f° → resolved=%{public}@ (native_used=%d) engineMode=%{public}@",
|
|
708
|
+
captureOrientation,
|
|
709
|
+
nativeResult.rollDegrees,
|
|
710
|
+
resolvedOrientation,
|
|
711
|
+
nativeResult.hadFrame ? Int32(1) : Int32(0),
|
|
712
|
+
engineMode)
|
|
713
|
+
// V15 — engine modes:
|
|
714
|
+
// 'hybrid' → hybrid engine, planar projection by default
|
|
715
|
+
// 'slitscan-rotate' → slit-scan, rectilinear, V13.0a + 1D NCC
|
|
716
|
+
// 'slitscan-both' → slit-scan, rectilinear, V13.0a + no gate
|
|
717
|
+
// + feather blend (iterate via overrides)
|
|
718
|
+
// Backward compat in -[RLISStitcherConfig configForMode:] handles
|
|
719
|
+
// 'firstwins-rectilinear' → 'slitscan-rotate' and warns on
|
|
720
|
+
// legacy 'firstwins' / 'firstwins-zoomed' / 'slitscan' modes.
|
|
721
|
+
let normalisedMode: String
|
|
722
|
+
switch engineMode {
|
|
723
|
+
case "hybrid": normalisedMode = "hybrid"
|
|
724
|
+
case "batch-keyframe":
|
|
725
|
+
// V16 Phase 1 — new mode. Skips the live incremental
|
|
726
|
+
// engines entirely; KeyframeGate accumulates accepted
|
|
727
|
+
// frames as JPEGs, on finalize OpenCVStitcher does the
|
|
728
|
+
// full-pipeline stitch.
|
|
729
|
+
normalisedMode = "batch-keyframe"
|
|
730
|
+
case "slitscan-rotate", "firstwins-rectilinear":
|
|
731
|
+
normalisedMode = "slitscan-rotate"
|
|
732
|
+
case "slitscan-both":
|
|
733
|
+
normalisedMode = "slitscan-both"
|
|
734
|
+
case "firstwins", "firstwins-zoomed", "slitscan":
|
|
735
|
+
NSLog("[V15-bridge] DEPRECATED engine '\(engineMode)' — using slitscan-both")
|
|
736
|
+
normalisedMode = "slitscan-both"
|
|
737
|
+
default:
|
|
738
|
+
NSLog("[V15-bridge] unknown engine '\(engineMode)' — using slitscan-both")
|
|
739
|
+
normalisedMode = "slitscan-both"
|
|
740
|
+
}
|
|
741
|
+
|
|
742
|
+
let useBatchKeyframe = (normalisedMode == "batch-keyframe")
|
|
743
|
+
let useFirstwinsClass = normalisedMode.hasPrefix("slitscan")
|
|
744
|
+
|
|
745
|
+
// Build the V15 config: factory default for the mode, then apply
|
|
746
|
+
// JS-side overrides.
|
|
747
|
+
let config = RLISStitcherConfig(forMode: normalisedMode)
|
|
748
|
+
Self.applyConfigOverrides(configOverrides, to: config)
|
|
749
|
+
|
|
750
|
+
if useBatchKeyframe {
|
|
751
|
+
// V16 Phase 1 — no live engine; spin up a keyframe
|
|
752
|
+
// collector that saves accepted frames to disk under
|
|
753
|
+
// Library/AppSupport/Captures/{uuid}/. On finalize
|
|
754
|
+
// these are handed to OpenCVStitcher's full pipeline
|
|
755
|
+
// (BA + GraphCut + ExposureComp + MultiBand) — the
|
|
756
|
+
// actually-quality path. Memory is bounded because
|
|
757
|
+
// KeyframeGate caps input at `keyframeMaxCount` (≤6).
|
|
758
|
+
do {
|
|
759
|
+
self.keyframeCollector = try OpenCVKeyframeCollector()
|
|
760
|
+
} catch {
|
|
761
|
+
NSLog("[V16-batch-keyframe] collector init failed: \(error.localizedDescription)")
|
|
762
|
+
self.keyframeCollector = nil
|
|
763
|
+
}
|
|
764
|
+
self.keyframePaths = []
|
|
765
|
+
self.keyframePoses = []
|
|
766
|
+
// V16 Phase 1.fix1 — keep frames in native landscape
|
|
767
|
+
// sensor orientation for the batch-keyframe path so the
|
|
768
|
+
// pose's intrinsics (which describe the unrotated
|
|
769
|
+
// 1920×1440 sensor) match the saved-image dimensions.
|
|
770
|
+
// The slit-scan and hybrid engines continue to receive
|
|
771
|
+
// `frameRotationDegrees` unchanged.
|
|
772
|
+
self.keyframeRotationDegrees = 0
|
|
773
|
+
// 2026-05-18 (Issue #1a fix) — keyframe EXIF Orientation
|
|
774
|
+
// is hardcoded to 6 ("rotate 90° CW for display") regardless
|
|
775
|
+
// of physical capture orientation.
|
|
776
|
+
//
|
|
777
|
+
// RCA: the earlier V16 Phase 1.fix2 mapping branched on
|
|
778
|
+
// `frameRotationDegrees` to "encode the user's perceived
|
|
779
|
+
// orientation". That was written for a hypothetical
|
|
780
|
+
// app whose orientation locks WITH the user's hold. Our
|
|
781
|
+
// host app is PORTRAIT-LOCKED: the rendering surface
|
|
782
|
+
// never rotates, regardless of how the operator holds
|
|
783
|
+
// the phone.
|
|
784
|
+
//
|
|
785
|
+
// RN's <Image> + Files.app honour EXIF when rendering.
|
|
786
|
+
// Sensor-native pixels are landscape-aspect (long axis =
|
|
787
|
+
// phone-Y). In a portrait-locked UI, displaying with
|
|
788
|
+
// EXIF=1 leaves the JPEG in landscape pixels rendered
|
|
789
|
+
// INTO portrait UI — the operator, head tilted to view
|
|
790
|
+
// their landscape capture, then sees the band thumbnails
|
|
791
|
+
// rotated 90° from their world view ("sideways"). Pre-
|
|
792
|
+
// bug-fix, the broken useDeviceOrientation hook always
|
|
793
|
+
// reported 'portrait' so `frameRotationDegrees=90` was
|
|
794
|
+
// always selected and EXIF=6 was always written — the
|
|
795
|
+
// operator never saw the misalignment because EXIF=6's
|
|
796
|
+
// 90° CW display rotation cancelled their physical 90°
|
|
797
|
+
// CCW head tilt in landscape-left view. Fixing the hook
|
|
798
|
+
// exposed the underlying portrait-lock mismatch.
|
|
799
|
+
//
|
|
800
|
+
// EXIF=6 (always) keeps the band thumbnails consistent
|
|
801
|
+
// with the portrait-locked UI in every hold. The FINAL
|
|
802
|
+
// panorama bake is independent — it consumes
|
|
803
|
+
// `config.captureOrientation` in cpp/stitcher.cpp's
|
|
804
|
+
// bake_rotation pass and is unaffected by this constant.
|
|
805
|
+
//
|
|
806
|
+
// If we ever unlock the host app's orientation (so the
|
|
807
|
+
// UI rotates with the user), this should revert to the
|
|
808
|
+
// 4-way switch. Tracked as a follow-up in the design
|
|
809
|
+
// doc.
|
|
810
|
+
self.keyframeExifOrientation = 6
|
|
811
|
+
// V16 Phase 1.fix4 — read cv::Stitcher knobs from JS config.
|
|
812
|
+
// Defaults to "plane" / "multiband" / "graphcut" — the
|
|
813
|
+
// proven combo cv::Stitcher::PANORAMA uses internally.
|
|
814
|
+
// Operator can A/B different warpers from the modal's
|
|
815
|
+
// Projection / Blender / Seam-finder sections.
|
|
816
|
+
self.batchWarperType =
|
|
817
|
+
(configOverrides["warperType"] as? String) ?? "plane"
|
|
818
|
+
self.batchBlenderType =
|
|
819
|
+
(configOverrides["blenderType"] as? String) ?? "multiband"
|
|
820
|
+
self.batchSeamFinderType =
|
|
821
|
+
(configOverrides["seamFinderType"] as? String) ?? "graphcut"
|
|
822
|
+
// V16 Phase 1b.fix5c — read inscribed-rect toggle. Defaults
|
|
823
|
+
// to FALSE if not provided by JS.
|
|
824
|
+
self.batchEnableInscribedRectCrop =
|
|
825
|
+
(configOverrides["enableMaxInscribedRectCrop"] as? Bool) ?? false
|
|
826
|
+
self.batchKeyframeMode = true
|
|
827
|
+
self.hybridEngine = nil
|
|
828
|
+
self.firstwinsEngine = nil
|
|
829
|
+
os_log(.fault, log: Self.diagLog,
|
|
830
|
+
"[V16-batch-keyframe] start mode=batch-keyframe rotation=0 (was %d, forced to 0 to match pose intrinsics) sessionDir=%{public}@",
|
|
831
|
+
frameRotationDegrees,
|
|
832
|
+
self.keyframeCollector?.sessionDir ?? "(nil)")
|
|
833
|
+
} else if useFirstwinsClass {
|
|
834
|
+
// Slit-scan engine always uses rectilinear in V15
|
|
835
|
+
// (firstwins-cylindrical and firstwins-zoomed modes were
|
|
836
|
+
// removed; their behaviour is unused).
|
|
837
|
+
self.firstwinsEngine = OpenCVFirstWinsCylindricalStitcher(
|
|
838
|
+
composeWidth: composeWidth,
|
|
839
|
+
composeHeight: composeHeight,
|
|
840
|
+
canvasWidth: canvasWidth,
|
|
841
|
+
canvasHeight: canvasHeight,
|
|
842
|
+
featherPx: featherPx,
|
|
843
|
+
frameRotationDegrees: frameRotationDegrees,
|
|
844
|
+
useRectilinear: true
|
|
845
|
+
)
|
|
846
|
+
self.firstwinsEngine?.setConfig(config)
|
|
847
|
+
self.hybridEngine = nil
|
|
848
|
+
self.batchKeyframeMode = false
|
|
849
|
+
} else {
|
|
850
|
+
self.hybridEngine = OpenCVIncrementalStitcher(
|
|
851
|
+
composeWidth: composeWidth,
|
|
852
|
+
composeHeight: composeHeight,
|
|
853
|
+
canvasWidth: canvasWidth,
|
|
854
|
+
canvasHeight: canvasHeight,
|
|
855
|
+
featherPx: featherPx,
|
|
856
|
+
frameRotationDegrees: frameRotationDegrees
|
|
857
|
+
)
|
|
858
|
+
self.hybridEngine?.setConfig(config)
|
|
859
|
+
self.firstwinsEngine = nil
|
|
860
|
+
self.batchKeyframeMode = false
|
|
861
|
+
}
|
|
862
|
+
self.isRunning = true
|
|
863
|
+
self.snapshotJpegQuality = max(1, min(100, snapshotJpegQuality))
|
|
864
|
+
self.snapshotEveryNAccepts = max(1, snapshotEveryNAccepts)
|
|
865
|
+
self.acceptsSinceSnapshot = 0
|
|
866
|
+
self.droppedBackpressure = 0
|
|
867
|
+
self.lastState = nil
|
|
868
|
+
// V15.0b — re-arm plane propagation for the new capture.
|
|
869
|
+
self.havePropagatedPlane = false
|
|
870
|
+
// V13.0c.1 — reset translation diagnostic state for the
|
|
871
|
+
// new capture. First-frame translation will be captured
|
|
872
|
+
// on the next consumeFrame call.
|
|
873
|
+
self.hasFirstFrameTranslation = false
|
|
874
|
+
self.consumeFrameCounter = 0
|
|
875
|
+
|
|
876
|
+
// V16 — configure the pose-driven keyframe gate from JS
|
|
877
|
+
// config overrides. Defaults match the field-tested values
|
|
878
|
+
// for a 90° landscape pan over a retail shelf: 40% required
|
|
879
|
+
// new content per keyframe, capped at 6 keyframes per
|
|
880
|
+
// capture. Values out of range are clamped silently.
|
|
881
|
+
let frameMode = (configOverrides["frameSelectionMode"] as? String)
|
|
882
|
+
?? "time-based"
|
|
883
|
+
// V16 A2 — both 'pose-based' and 'flow-based' enable the gate;
|
|
884
|
+
// they differ only in the novelty metric (plane-overlap vs
|
|
885
|
+
// sparse-flow). 'time-based' = passthrough.
|
|
886
|
+
self.keyframeGate.enabled =
|
|
887
|
+
(frameMode == "pose-based" || frameMode == "flow-based")
|
|
888
|
+
self.keyframeGate.strategy =
|
|
889
|
+
(frameMode == "flow-based") ? .flow : .pose
|
|
890
|
+
if let v = configOverrides["keyframeOverlapThreshold"] as? Double {
|
|
891
|
+
self.keyframeGate.overlapThreshold = max(0.10, min(0.80, v))
|
|
892
|
+
} else {
|
|
893
|
+
self.keyframeGate.overlapThreshold = 0.4
|
|
894
|
+
}
|
|
895
|
+
if let v = configOverrides["keyframeMaxCount"] as? Int {
|
|
896
|
+
self.keyframeGate.maxCount = max(3, min(10, v))
|
|
897
|
+
} else {
|
|
898
|
+
self.keyframeGate.maxCount = 6
|
|
899
|
+
}
|
|
900
|
+
// V16 A2 — flow tuning. C++ side also clamps defensively
|
|
901
|
+
// (setFlowMaxCorners ≥ 30, setFlowQualityLevel ∈ (0, 1],
|
|
902
|
+
// setFlowMinDistance ≥ 1.0) so we layer the JS-side modal
|
|
903
|
+
// ranges over those minimum invariants. Reading these
|
|
904
|
+
// unconditionally — they're cheap and the gate ignores them
|
|
905
|
+
// when strategy != .flow.
|
|
906
|
+
if let v = configOverrides["flowMaxCorners"] as? Int {
|
|
907
|
+
self.keyframeGate.flowMaxCorners = max(50, min(300, v))
|
|
908
|
+
} else {
|
|
909
|
+
self.keyframeGate.flowMaxCorners = 150
|
|
910
|
+
}
|
|
911
|
+
if let v = configOverrides["flowQualityLevel"] as? Double {
|
|
912
|
+
self.keyframeGate.flowQualityLevel = max(0.005, min(0.05, v))
|
|
913
|
+
} else {
|
|
914
|
+
self.keyframeGate.flowQualityLevel = 0.01
|
|
915
|
+
}
|
|
916
|
+
if let v = configOverrides["flowMinDistance"] as? Double {
|
|
917
|
+
self.keyframeGate.flowMinDistance = max(1.0, min(50.0, v))
|
|
918
|
+
} else if let v = configOverrides["flowMinDistance"] as? Int {
|
|
919
|
+
// JS sometimes ships Ints when the value happens to be
|
|
920
|
+
// an integer (SegmentedControl options are strings that
|
|
921
|
+
// parseInt into Ints). Accept either shape.
|
|
922
|
+
self.keyframeGate.flowMinDistance = max(1.0, min(50.0, Double(v)))
|
|
923
|
+
} else {
|
|
924
|
+
self.keyframeGate.flowMinDistance = 10.0
|
|
925
|
+
}
|
|
926
|
+
// V16 — translation-budget force-accept (Flow strategy). cm
|
|
927
|
+
// on the JS side (UI-friendly), converted to metres by the
|
|
928
|
+
// KeyframeGate.swift setter. Clamp to [0, 100] cm at start so
|
|
929
|
+
// a stray JS default can't put the gate in an unworkable
|
|
930
|
+
// state. Default 0 = disabled (preserves pre-V16 behaviour
|
|
931
|
+
// for callers that don't opt in).
|
|
932
|
+
if let v = configOverrides["flowMaxTranslationCm"] as? Double {
|
|
933
|
+
self.keyframeGate.flowMaxTranslationCm = max(0.0, min(100.0, v))
|
|
934
|
+
} else if let v = configOverrides["flowMaxTranslationCm"] as? Int {
|
|
935
|
+
self.keyframeGate.flowMaxTranslationCm = max(0.0, min(100.0, Double(v)))
|
|
936
|
+
} else {
|
|
937
|
+
self.keyframeGate.flowMaxTranslationCm = 0.0
|
|
938
|
+
}
|
|
939
|
+
// V16 — novelty aggregation percentile. Clamp at start to
|
|
940
|
+
// [0.5, 0.99]; the bridge re-clamps but matching it here
|
|
941
|
+
// means our state stays in-range for logging. Default 0.85
|
|
942
|
+
// — picks up leading-edge motion sooner than the pre-V16
|
|
943
|
+
// median (0.5).
|
|
944
|
+
if let v = configOverrides["flowNoveltyPercentile"] as? Double {
|
|
945
|
+
self.keyframeGate.flowNoveltyPercentile = max(0.5, min(0.99, v))
|
|
946
|
+
} else {
|
|
947
|
+
self.keyframeGate.flowNoveltyPercentile = 0.85
|
|
948
|
+
}
|
|
949
|
+
// V16 — Swift-side eval throttle. Default 1 (every consumeFrame
|
|
950
|
+
// runs evaluate). Range 1-10. At 1, identical to pre-V16
|
|
951
|
+
// behaviour; at higher N, evaluate runs every Nth frame to
|
|
952
|
+
// save CPU/battery on long captures. Doesn't change WHICH
|
|
953
|
+
// frames are accepted (still subject to overlapThreshold +
|
|
954
|
+
// translation budget) — just samples less frequently.
|
|
955
|
+
if let v = configOverrides["flowEvalEveryNFrames"] as? Int {
|
|
956
|
+
self.keyframeGate.flowEvalEveryNFrames = max(1, min(10, v))
|
|
957
|
+
} else {
|
|
958
|
+
self.keyframeGate.flowEvalEveryNFrames = 1
|
|
959
|
+
}
|
|
960
|
+
self.keyframeGate.reset()
|
|
961
|
+
os_log(.fault, log: Self.diagLog,
|
|
962
|
+
"[V16-keyframe] start gate enabled=%d strategy=%{public}@ thr=%.2f max=%d flow(maxCorners=%d quality=%.3f minDist=%.1f maxTransCm=%.1f pctile=%.2f evalEveryN=%d)",
|
|
963
|
+
self.keyframeGate.enabled ? 1 : 0,
|
|
964
|
+
self.keyframeGate.strategy == .flow ? "flow" : "pose",
|
|
965
|
+
self.keyframeGate.overlapThreshold,
|
|
966
|
+
self.keyframeGate.maxCount,
|
|
967
|
+
self.keyframeGate.flowMaxCorners,
|
|
968
|
+
self.keyframeGate.flowQualityLevel,
|
|
969
|
+
self.keyframeGate.flowMinDistance,
|
|
970
|
+
self.keyframeGate.flowMaxTranslationCm,
|
|
971
|
+
self.keyframeGate.flowNoveltyPercentile,
|
|
972
|
+
Int32(self.keyframeGate.flowEvalEveryNFrames))
|
|
973
|
+
|
|
974
|
+
stateLock.unlock()
|
|
975
|
+
|
|
976
|
+
// Register with the AR session — only when running in the
|
|
977
|
+
// AR-frame-stream-driven mode. In jsDriver mode (iOS non-AR
|
|
978
|
+
// captures) the AR session is intentionally stopped so the
|
|
979
|
+
// vision-camera holds the camera; frames arrive via
|
|
980
|
+
// processFrameAtPath from JS instead. Registering as the
|
|
981
|
+
// consumer here would either crash (no running session) or
|
|
982
|
+
// mis-route frames once an AR session somewhere else came up.
|
|
983
|
+
if frameSourceMode != "jsDriver" {
|
|
984
|
+
RNSARSession.shared.incrementalConsumer = self
|
|
985
|
+
}
|
|
986
|
+
}
|
|
987
|
+
|
|
988
|
+
/// Stop ingestion + write the final panorama to `outputPath`.
|
|
989
|
+
/// Returns the result on the main thread via completion.
|
|
990
|
+
///
|
|
991
|
+
/// V12.1 frame-leak fix: the previous version waited until the
|
|
992
|
+
/// finalize block ran on the work queue to set isRunning=false.
|
|
993
|
+
/// Between the JS shutter-release and that block running, the
|
|
994
|
+
/// AR delegate could deliver several more frames — each one
|
|
995
|
+
/// passed consumeFrame's `isRunning == true` check and got
|
|
996
|
+
/// ingested into the canvas, producing visible "phantom" frames
|
|
997
|
+
/// V15 — apply per-stage JS overrides on top of a mode default.
|
|
998
|
+
/// Keys recognised in `overrides`: any non-readonly RLISStitcherConfig
|
|
999
|
+
/// field. Unrecognised keys are ignored. Values out of range are
|
|
1000
|
+
/// clamped silently (e.g. kPanAxisFractionRect outside [0.05, 0.90]).
|
|
1001
|
+
private static func applyConfigOverrides(_ overrides: [String: Any],
|
|
1002
|
+
to config: RLISStitcherConfig) {
|
|
1003
|
+
if let v = overrides["kPanAxisFractionRect"] as? Double {
|
|
1004
|
+
config.kPanAxisFractionRect = max(0.05, min(0.90, v))
|
|
1005
|
+
}
|
|
1006
|
+
if let v = overrides["kMinAcceptDeltaPx"] as? Int {
|
|
1007
|
+
config.kMinAcceptDeltaPx = max(0, min(500, v))
|
|
1008
|
+
}
|
|
1009
|
+
if let v = overrides["enableTriangulation"] as? Bool {
|
|
1010
|
+
config.enableTriangulation = v
|
|
1011
|
+
}
|
|
1012
|
+
if let v = overrides["enableTriAccumulator"] as? Bool {
|
|
1013
|
+
config.enableTriAccumulator = v
|
|
1014
|
+
}
|
|
1015
|
+
if let v = overrides["enable1dNcc"] as? Bool {
|
|
1016
|
+
config.enable1dNcc = v
|
|
1017
|
+
}
|
|
1018
|
+
if let v = overrides["nccSearchRadius1d"] as? Int {
|
|
1019
|
+
config.nccSearchRadius1d = max(5, min(60, v))
|
|
1020
|
+
}
|
|
1021
|
+
if let v = overrides["enable2dNcc"] as? Bool {
|
|
1022
|
+
config.enable2dNcc = v
|
|
1023
|
+
}
|
|
1024
|
+
if let v = overrides["enableRansacHomography"] as? Bool {
|
|
1025
|
+
config.enableRansacHomography = v
|
|
1026
|
+
}
|
|
1027
|
+
if let v = overrides["paintMode"] as? String {
|
|
1028
|
+
switch v {
|
|
1029
|
+
case "FirstPaintedWins": config.paintMode = .firstPaintedWins
|
|
1030
|
+
case "FeatherBlend": config.paintMode = .featherBlend
|
|
1031
|
+
default: break
|
|
1032
|
+
}
|
|
1033
|
+
}
|
|
1034
|
+
if let v = overrides["hybridProjection"] as? String {
|
|
1035
|
+
switch v {
|
|
1036
|
+
case "Cylindrical": config.hybridProjection = .cylindrical
|
|
1037
|
+
case "Planar": config.hybridProjection = .planar
|
|
1038
|
+
default: break
|
|
1039
|
+
}
|
|
1040
|
+
}
|
|
1041
|
+
if let v = overrides["useDetectedPlane"] as? Bool {
|
|
1042
|
+
config.useDetectedPlane = v
|
|
1043
|
+
}
|
|
1044
|
+
if let v = overrides["sliverPosition"] as? String {
|
|
1045
|
+
switch v {
|
|
1046
|
+
case "Center": config.sliverPosition = .center
|
|
1047
|
+
case "Bottom": config.sliverPosition = .bottom
|
|
1048
|
+
case "Top": config.sliverPosition = .top
|
|
1049
|
+
default: break
|
|
1050
|
+
}
|
|
1051
|
+
}
|
|
1052
|
+
if let v = overrides["firstFrameFullFrame"] as? Bool {
|
|
1053
|
+
config.firstFrameFullFrame = v
|
|
1054
|
+
}
|
|
1055
|
+
// V15.0d new overrides.
|
|
1056
|
+
if let v = overrides["planeSource"] as? String {
|
|
1057
|
+
switch v {
|
|
1058
|
+
case "Disabled": config.planeSource = .disabled
|
|
1059
|
+
case "ARKitDetected": config.planeSource = .arKitDetected
|
|
1060
|
+
case "Virtual": config.planeSource = .virtual
|
|
1061
|
+
default: break
|
|
1062
|
+
}
|
|
1063
|
+
}
|
|
1064
|
+
if let v = overrides["virtualPlaneDepthMeters"] as? Double {
|
|
1065
|
+
config.virtualPlaneDepthMeters = max(0.3, min(5.0, v))
|
|
1066
|
+
}
|
|
1067
|
+
if let v = overrides["arkitPlaneAlignmentThreshold"] as? Double {
|
|
1068
|
+
config.arkitPlaneAlignmentThreshold = max(0.0, min(1.0, v))
|
|
1069
|
+
}
|
|
1070
|
+
if let v = overrides["planeProjectionStyle"] as? String {
|
|
1071
|
+
switch v {
|
|
1072
|
+
case "Trapezoidal": config.planeProjectionStyle = .trapezoidal
|
|
1073
|
+
case "Rectified": config.planeProjectionStyle = .rectified
|
|
1074
|
+
default: break
|
|
1075
|
+
}
|
|
1076
|
+
}
|
|
1077
|
+
if let v = overrides["nccSearchMargin2d"] as? Int {
|
|
1078
|
+
config.nccSearchMargin2d = max(4, min(60, v))
|
|
1079
|
+
}
|
|
1080
|
+
if let v = overrides["nccConfidenceThreshold2d"] as? Double {
|
|
1081
|
+
config.nccConfidenceThreshold2d = max(0.30, min(0.99, v))
|
|
1082
|
+
}
|
|
1083
|
+
if let v = overrides["enableNcc2dEmaSmoothing"] as? Bool {
|
|
1084
|
+
config.enableNcc2dEmaSmoothing = v
|
|
1085
|
+
}
|
|
1086
|
+
if let v = overrides["ncc2dEmaAlpha"] as? Double {
|
|
1087
|
+
config.ncc2dEmaAlpha = max(0.05, min(0.95, v))
|
|
1088
|
+
}
|
|
1089
|
+
if let v = overrides["enableNcc2dPanAxisLock"] as? Bool {
|
|
1090
|
+
config.enableNcc2dPanAxisLock = v
|
|
1091
|
+
}
|
|
1092
|
+
if let v = overrides["ncc2dCrossAxisLockPx"] as? Int {
|
|
1093
|
+
config.ncc2dCrossAxisLockPx = max(0, min(30, v))
|
|
1094
|
+
}
|
|
1095
|
+
// Propagate the alignment threshold to the AR session so its
|
|
1096
|
+
// didAdd / didUpdate filter uses the operator-chosen value.
|
|
1097
|
+
// (planeAlignmentThreshold is a Float on the AR session.)
|
|
1098
|
+
RNSARSession.shared.planeAlignmentThreshold =
|
|
1099
|
+
Float(config.arkitPlaneAlignmentThreshold)
|
|
1100
|
+
}
|
|
1101
|
+
|
|
1102
|
+
/// after the user thought they had released. The engine refs
|
|
1103
|
+
/// and isRunning flag are now flipped SYNCHRONOUSLY here so the
|
|
1104
|
+
/// AR delegate's very next consumeFrame sees isRunning=false.
|
|
1105
|
+
/// The work-queue body just runs the engine's own finalize.
|
|
1106
|
+
@objc public func finalize(
|
|
1107
|
+
toPath outputPath: String,
|
|
1108
|
+
jpegQuality: Int,
|
|
1109
|
+
completion: @escaping ([String: Any]?, NSError?) -> Void
|
|
1110
|
+
) {
|
|
1111
|
+
// V12.9 fix #3 — flip isRunning=false BEFORE nulling the AR
|
|
1112
|
+
// consumer. The previous order had a race window: AR
|
|
1113
|
+
// delegate captures `incrementalConsumer` (non-nil), the
|
|
1114
|
+
// delegate is briefly suspended, finalize runs and sets
|
|
1115
|
+
// consumer=nil + isRunning=false, then the suspended delegate
|
|
1116
|
+
// resumes and calls consumer.consumeFrame(). consumeFrame
|
|
1117
|
+
// saw isRunning=false at its first guard and bailed in MOST
|
|
1118
|
+
// cases, but the in-flight workQueue task (if any) had
|
|
1119
|
+
// already captured non-nil engine refs at consumeFrame
|
|
1120
|
+
// entry — by re-checking inside the workQueue async we
|
|
1121
|
+
// catch it, but only just. Flipping isRunning first
|
|
1122
|
+
// collapses the race: any consumeFrame entered after this
|
|
1123
|
+
// line sees isRunning=false at its very first guard.
|
|
1124
|
+
stateLock.lock()
|
|
1125
|
+
let hybrid = self.hybridEngine
|
|
1126
|
+
let slit = self.firstwinsEngine
|
|
1127
|
+
let inBatchKeyframeMode = self.batchKeyframeMode
|
|
1128
|
+
let collector = self.keyframeCollector
|
|
1129
|
+
let paths = self.keyframePaths
|
|
1130
|
+
// V16 Phase 1b.fix4 — snapshot the cv::Stitcher knobs and
|
|
1131
|
+
// EXIF orientation under stateLock so the workQueue closure
|
|
1132
|
+
// has a stable view of these values, independent of any
|
|
1133
|
+
// concurrent start() that may begin a new capture before
|
|
1134
|
+
// this closure finishes.
|
|
1135
|
+
//
|
|
1136
|
+
// Why this matters (RCA from Sentry crashes 2026-05-09
|
|
1137
|
+
// 21:59-22:03, all 3 .ips traces):
|
|
1138
|
+
// EXC_BAD_ACCESS at objc_retain+16, frame 1 = closure #1
|
|
1139
|
+
// in finalize+2648, queue = com.retailens.incremental.
|
|
1140
|
+
// stitcher. +2648 lands inside the os_log call that
|
|
1141
|
+
// bridges self.batchWarperType → NSString via
|
|
1142
|
+
// swift_bridgeObjectRetain → objc_retain. The retain
|
|
1143
|
+
// loaded a torn buffer pointer because:
|
|
1144
|
+
//
|
|
1145
|
+
// T0: finalize releases stateLock, dispatches workQueue
|
|
1146
|
+
// async closure (long stitch ahead).
|
|
1147
|
+
// T1: User sees fix2's "9002 No active capture" popup
|
|
1148
|
+
// (race between shutter-release + auto-finalize
|
|
1149
|
+
// useEffect, fixed in fix3 but pre-existing in
|
|
1150
|
+
// fix2).
|
|
1151
|
+
// T2: User dismisses popup, starts a new capture.
|
|
1152
|
+
// T3: start() acquires stateLock, sees isRunning==false,
|
|
1153
|
+
// WRITES self.batchWarperType = newValue. ARC
|
|
1154
|
+
// releases the old String buffer.
|
|
1155
|
+
// T4: workQueue closure (still mid-finalize from
|
|
1156
|
+
// capture N) loads self.batchWarperType for os_log.
|
|
1157
|
+
// Read interleaved with T3's write → torn String
|
|
1158
|
+
// buffer pointer → swift_bridgeObjectRetain →
|
|
1159
|
+
// objc_retain on freed memory → KERN_INVALID_ADDRESS.
|
|
1160
|
+
//
|
|
1161
|
+
// Fix3's JS dedupe closes the practical trigger (no popup,
|
|
1162
|
+
// no operator-confusion-driven new capture). Fix4 closes
|
|
1163
|
+
// the underlying race regardless of UI flow correctness.
|
|
1164
|
+
let batchWarperType = self.batchWarperType
|
|
1165
|
+
let batchBlenderType = self.batchBlenderType
|
|
1166
|
+
let batchSeamFinderType = self.batchSeamFinderType
|
|
1167
|
+
let batchEnableInscribedRectCrop = self.batchEnableInscribedRectCrop
|
|
1168
|
+
let keyframeExifOrientation = self.keyframeExifOrientation
|
|
1169
|
+
// AR-STITCHING-TWO-MODES — see memory/ar-stitching-two-modes.md
|
|
1170
|
+
// Snapshot the capture-time hold orientation for the bake-
|
|
1171
|
+
// rotation pass. Reasons same as the other snapshots above:
|
|
1172
|
+
// stateLock-protected against a concurrent start() rewriting
|
|
1173
|
+
// self.captureOrientation while the workQueue closure is mid-
|
|
1174
|
+
// flight.
|
|
1175
|
+
let captureOrientation = self.captureOrientation
|
|
1176
|
+
// V16 Phase 1.fix4 — pose array still held on self ivar (and
|
|
1177
|
+
// queued for persistent sidecar storage in a future debug-menu
|
|
1178
|
+
// feature) but NOT passed to the stitcher in this drop. fix4
|
|
1179
|
+
// uses feature-matched stitchFramePaths which doesn't take
|
|
1180
|
+
// poses; cv::Stitcher's BA derives camera placement from
|
|
1181
|
+
// features. Drop the closure-capture to avoid a compile
|
|
1182
|
+
// warning; ARKit pose data is preserved on the ivar regardless.
|
|
1183
|
+
_ = self.keyframePoses
|
|
1184
|
+
self.hybridEngine = nil
|
|
1185
|
+
self.firstwinsEngine = nil
|
|
1186
|
+
self.batchKeyframeMode = false
|
|
1187
|
+
self.keyframeCollector = nil
|
|
1188
|
+
self.keyframePaths = []
|
|
1189
|
+
self.keyframePoses = []
|
|
1190
|
+
self.isRunning = false
|
|
1191
|
+
let drops = self.droppedBackpressure
|
|
1192
|
+
stateLock.unlock()
|
|
1193
|
+
|
|
1194
|
+
// V16 Phase 1b.fix8 (C2 — stateless finalize):
|
|
1195
|
+
// The prior 7 fix attempts all individually snapshotted the
|
|
1196
|
+
// batch* ivars + paths/collector/engines/drops above into
|
|
1197
|
+
// closure-locals. This worked at the source level but the
|
|
1198
|
+
// `objc_retain` family of crashes kept moving from one
|
|
1199
|
+
// closure to the next (closure #1 → closure #2, offset
|
|
1200
|
+
// +2648 → +5176) — the compiler still had implicit-capture
|
|
1201
|
+
// latitude. C2 closes the entire class:
|
|
1202
|
+
// 1. Bundle every snapshot into a value-typed
|
|
1203
|
+
// FinalizePayload (defined at file scope above the
|
|
1204
|
+
// class declaration).
|
|
1205
|
+
// 2. Pass the payload BY VALUE into the workQueue closure
|
|
1206
|
+
// via an explicit capture list `[payload, completion]`.
|
|
1207
|
+
// 3. Bracket the closure with `C2-INVARIANT` marker
|
|
1208
|
+
// comments and enforce "no `self.*` inside" via the
|
|
1209
|
+
// CI test at scripts/check_c2_invariant.sh.
|
|
1210
|
+
// Net effect: the closure literally cannot read any
|
|
1211
|
+
// stitcher ivar. Any future edit that re-introduces a
|
|
1212
|
+
// `self.` reference inside the closure is caught at CI.
|
|
1213
|
+
let arWasRunning = inBatchKeyframeMode
|
|
1214
|
+
&& RNSARSession.shared.isRunning
|
|
1215
|
+
let cleaned = (outputPath.hasPrefix("file://"))
|
|
1216
|
+
? String(outputPath.dropFirst(7))
|
|
1217
|
+
: outputPath
|
|
1218
|
+
let q = max(1, min(100, jpegQuality))
|
|
1219
|
+
let payload = FinalizePayload(
|
|
1220
|
+
cleaned: cleaned,
|
|
1221
|
+
q: q,
|
|
1222
|
+
inBatchKeyframeMode: inBatchKeyframeMode,
|
|
1223
|
+
hybrid: hybrid,
|
|
1224
|
+
slit: slit,
|
|
1225
|
+
collector: collector,
|
|
1226
|
+
paths: paths,
|
|
1227
|
+
batchWarperType: batchWarperType,
|
|
1228
|
+
batchBlenderType: batchBlenderType,
|
|
1229
|
+
batchSeamFinderType: batchSeamFinderType,
|
|
1230
|
+
batchEnableInscribedRectCrop: batchEnableInscribedRectCrop,
|
|
1231
|
+
keyframeExifOrientation: keyframeExifOrientation,
|
|
1232
|
+
captureOrientation: captureOrientation,
|
|
1233
|
+
drops: drops,
|
|
1234
|
+
arWasRunning: arWasRunning
|
|
1235
|
+
)
|
|
1236
|
+
|
|
1237
|
+
// Then detach the AR consumer. Any in-flight delegate that
|
|
1238
|
+
// already captured the consumer reference will reach
|
|
1239
|
+
// consumeFrame, see isRunning=false, and bail.
|
|
1240
|
+
RNSARSession.shared.incrementalConsumer = nil
|
|
1241
|
+
|
|
1242
|
+
// V16 Phase 1b.fix1 — pause the AR session for the duration
|
|
1243
|
+
// of the stitch (batch-keyframe path only). ARSession holds
|
|
1244
|
+
// a pixel-buffer pool, world map, and plane geometry that
|
|
1245
|
+
// collectively contribute ~200-300 MB to baseline. Dropping
|
|
1246
|
+
// them while cv::Stitcher's BA + GraphCut + MultiBand runs
|
|
1247
|
+
// gives the stitcher more headroom under the per-process
|
|
1248
|
+
// limit. Restart on the main thread after the stitch
|
|
1249
|
+
// completes so the next capture has AR ready (next plane
|
|
1250
|
+
// detection + tracking re-initialise will take 2-3 s, which
|
|
1251
|
+
// matches Ram's chosen "Option C" trade-off).
|
|
1252
|
+
//
|
|
1253
|
+
// `arWasRunning` was computed above into FinalizePayload — read
|
|
1254
|
+
// from `payload.arWasRunning` here so we have one source of
|
|
1255
|
+
// truth (the value the closure's defer also reads).
|
|
1256
|
+
if payload.arWasRunning {
|
|
1257
|
+
os_log(.fault, log: Self.diagLog,
|
|
1258
|
+
"[V16-batch-keyframe] pausing AR session for stitch (memory drop)")
|
|
1259
|
+
RNSARSession.shared.stop()
|
|
1260
|
+
}
|
|
1261
|
+
|
|
1262
|
+
// V16 Phase 1b.fix6 — ARCHITECTURAL: workQueue.sync (not async)
|
|
1263
|
+
// for finalize.
|
|
1264
|
+
//
|
|
1265
|
+
// Background: finalize ran 5 prior fixes targeting an
|
|
1266
|
+
// EXC_BAD_ACCESS in objc_retain inside this closure. fix4
|
|
1267
|
+
// snapshotted every self.batch*, self.captureOrientation,
|
|
1268
|
+
// self.keyframePoses, and the engine refs into closure-locals
|
|
1269
|
+
// under stateLock, closing the visible torn-pointer race.
|
|
1270
|
+
// Three Sentry traces post-fix4 still showed the same crash
|
|
1271
|
+
// signature (frame 1 = closure #1 in finalize+N, queue =
|
|
1272
|
+
// com.retailens.incremental.stitcher), which per the
|
|
1273
|
+
// systematic-debugging skill (3+ fixes failed on the same
|
|
1274
|
+
// symptom = wrong architecture) means the workQueue.async
|
|
1275
|
+
// pattern itself is the problem, not any specific captured
|
|
1276
|
+
// ivar.
|
|
1277
|
+
//
|
|
1278
|
+
// Why .sync fixes the entire class of issues:
|
|
1279
|
+
// 1. Serializes finalize with start(): the bridge thread
|
|
1280
|
+
// can't return to JS until the stitch + completion fire.
|
|
1281
|
+
// JS can't trigger a new start() during the in-flight
|
|
1282
|
+
// stitch because the JS side awaits the finalize promise.
|
|
1283
|
+
// 2. Eliminates the "1-frame crash" race: with .async, when
|
|
1284
|
+
// paths.count==1 the closure rejected via completion +
|
|
1285
|
+
// returned synchronously, but the auto-finalize useEffect
|
|
1286
|
+
// on JS occasionally fired a SECOND finalize before the
|
|
1287
|
+
// first one's completion had landed. With .sync, the
|
|
1288
|
+
// second finalize is blocked until the first one returns.
|
|
1289
|
+
// 3. Removes the "what if completion's captured resolver/
|
|
1290
|
+
// rejecter get released by the bridge before the closure
|
|
1291
|
+
// runs" failure mode entirely — there's no longer any
|
|
1292
|
+
// window between dispatch and execution.
|
|
1293
|
+
//
|
|
1294
|
+
// Cost: bridge thread blocks for the stitch duration (2-5 s
|
|
1295
|
+
// on 4-6 keyframes at iPhone 16 Pro). The bridge thread is
|
|
1296
|
+
// NOT main (RCTEventEmitter.requiresMainQueueSetup() is false
|
|
1297
|
+
// and we don't override methodQueue) so UI stays responsive.
|
|
1298
|
+
// Other IncrementalStitcher bridge calls queue up
|
|
1299
|
+
// for ~3 s — acceptable since the JS side is awaiting the
|
|
1300
|
+
// finalize promise anyway and isn't issuing other calls
|
|
1301
|
+
// during that interval.
|
|
1302
|
+
//
|
|
1303
|
+
// Deadlock check: workQueue is consumed by consumeFrame
|
|
1304
|
+
// (AR delegate, line ~1424) and finalize. If the AR delegate
|
|
1305
|
+
// is currently mid-frame on workQueue when we call .sync, the
|
|
1306
|
+
// .sync waits for it (~50-100 ms JPEG encode) — not a
|
|
1307
|
+
// deadlock, just brief serialization. No other queue
|
|
1308
|
+
// dispatches synchronously TO workQueue, so .sync is safe.
|
|
1309
|
+
// MARK: C2-INVARIANT-START — no `self.` access below this line until C2-INVARIANT-END
|
|
1310
|
+
//
|
|
1311
|
+
// Enforced by scripts/check_c2_invariant.sh: any `self.` token
|
|
1312
|
+
// (non-comment) inside this region is a CI failure. Every
|
|
1313
|
+
// value the closure needs is plumbed through `payload`.
|
|
1314
|
+
//
|
|
1315
|
+
// Capture list is EXPLICIT — `[payload, completion]` ONLY.
|
|
1316
|
+
// The compiler will refuse any reference here that isn't
|
|
1317
|
+
// satisfied by these two captures, value-typed members of
|
|
1318
|
+
// `payload`, static types (`Self.diagLog`, `OpenCVStitcher`,
|
|
1319
|
+
// `FileManager`, `CGDataProvider`, `CGImage`, `NSError`,
|
|
1320
|
+
// `RNSARSession`), or local lets/declarations made
|
|
1321
|
+
// inside the closure.
|
|
1322
|
+
workQueue.sync { [payload, completion] in
|
|
1323
|
+
// V16 Phase 1b.fix1 — defer-restart AR session. Fires
|
|
1324
|
+
// on every exit path (success, error, early return).
|
|
1325
|
+
// Restart is dispatched to main because ARSession.run()
|
|
1326
|
+
// expects main-thread invocation to set up its rendering
|
|
1327
|
+
// hooks; happens AFTER the stitch completes so it doesn't
|
|
1328
|
+
// contend for the memory budget.
|
|
1329
|
+
defer {
|
|
1330
|
+
if payload.arWasRunning {
|
|
1331
|
+
// Inner closure body references only `Self.diagLog`
|
|
1332
|
+
// (static type) and `RNSARSession.shared`
|
|
1333
|
+
// (singleton) — both name-resolve without
|
|
1334
|
+
// capturing self. No explicit capture list
|
|
1335
|
+
// required; the C2 invariant script grep-checks
|
|
1336
|
+
// for `self.*` tokens which this body has none of.
|
|
1337
|
+
DispatchQueue.main.async {
|
|
1338
|
+
os_log(.fault, log: Self.diagLog,
|
|
1339
|
+
"[V16-batch-keyframe] restarting AR session post-stitch")
|
|
1340
|
+
RNSARSession.shared.start()
|
|
1341
|
+
}
|
|
1342
|
+
}
|
|
1343
|
+
}
|
|
1344
|
+
do {
|
|
1345
|
+
if payload.inBatchKeyframeMode {
|
|
1346
|
+
// V16 Phase 1 — hand collected keyframes + poses
|
|
1347
|
+
// to OpenCVStitcher's full BA + GraphCut +
|
|
1348
|
+
// ExposureComp + MultiBand pipeline. ≤6 frames
|
|
1349
|
+
// means BA stays bounded and MultiBand fits.
|
|
1350
|
+
os_log(.fault, log: Self.diagLog,
|
|
1351
|
+
"[V16-batch-keyframe] finalize ENTRY frames=%d",
|
|
1352
|
+
payload.paths.count)
|
|
1353
|
+
// V16 Phase 1b.fix7 — single-keyframe UX:
|
|
1354
|
+
// accept a 1-frame finalize by copying the lone
|
|
1355
|
+
// keyframe JPEG to outputPath (preserving the JPEG
|
|
1356
|
+
// bytes as-is, no re-encode). Previously this path
|
|
1357
|
+
// rejected with code 9003, and an auto-finalize
|
|
1358
|
+
// useEffect on JS could fire it during a quick
|
|
1359
|
+
// tap-and-release. The user-visible failure
|
|
1360
|
+
// ("9003 only 1 keyframe saved") was both
|
|
1361
|
+
// confusing AND occasionally racing with a second
|
|
1362
|
+
// finalize call into the EXC_BAD_ACCESS path that
|
|
1363
|
+
// fix4/fix6 closed. Returning the single frame as
|
|
1364
|
+
// the output is the right UX: a single keyframe IS
|
|
1365
|
+
// a valid panorama capture (just one shot).
|
|
1366
|
+
if payload.paths.count < 2 {
|
|
1367
|
+
if payload.paths.count == 1 {
|
|
1368
|
+
let src = payload.paths[0]
|
|
1369
|
+
do {
|
|
1370
|
+
// Remove any pre-existing file at the
|
|
1371
|
+
// output path — copyItem refuses to
|
|
1372
|
+
// overwrite, and a stale tmp file from
|
|
1373
|
+
// a prior auto-finalize attempt is the
|
|
1374
|
+
// common case.
|
|
1375
|
+
let fm = FileManager.default
|
|
1376
|
+
if fm.fileExists(atPath: payload.cleaned) {
|
|
1377
|
+
try fm.removeItem(atPath: payload.cleaned)
|
|
1378
|
+
}
|
|
1379
|
+
try fm.copyItem(atPath: src, toPath: payload.cleaned)
|
|
1380
|
+
// Read back the JPEG dimensions for
|
|
1381
|
+
// the result dictionary — match
|
|
1382
|
+
// OpenCVStitcher.stitchFramePaths'
|
|
1383
|
+
// {width, height} contract so JS
|
|
1384
|
+
// doesn't have to special-case a
|
|
1385
|
+
// single-frame result.
|
|
1386
|
+
var width: Int = 0
|
|
1387
|
+
var height: Int = 0
|
|
1388
|
+
if let provider = CGDataProvider(filename: payload.cleaned),
|
|
1389
|
+
let img = CGImage(
|
|
1390
|
+
jpegDataProviderSource: provider,
|
|
1391
|
+
decode: nil,
|
|
1392
|
+
shouldInterpolate: false,
|
|
1393
|
+
intent: .defaultIntent) {
|
|
1394
|
+
width = img.width
|
|
1395
|
+
height = img.height
|
|
1396
|
+
}
|
|
1397
|
+
os_log(.fault, log: Self.diagLog,
|
|
1398
|
+
"[V16-batch-keyframe.fix7] single-keyframe finalize: copied %{public}@ → %{public}@ (%dx%d)",
|
|
1399
|
+
src, payload.cleaned,
|
|
1400
|
+
Int32(width), Int32(height))
|
|
1401
|
+
completion([
|
|
1402
|
+
"panoramaPath": payload.cleaned,
|
|
1403
|
+
"width": width,
|
|
1404
|
+
"height": height,
|
|
1405
|
+
"acceptedCount": 1,
|
|
1406
|
+
"droppedBackpressure": payload.drops,
|
|
1407
|
+
"batchKeyframeSessionDir":
|
|
1408
|
+
payload.collector?.sessionDir ?? "",
|
|
1409
|
+
"batchKeyframeCount": 1,
|
|
1410
|
+
"singleKeyframe": true,
|
|
1411
|
+
], nil)
|
|
1412
|
+
return
|
|
1413
|
+
} catch let copyErr as NSError {
|
|
1414
|
+
// Fall through to the legacy "not
|
|
1415
|
+
// enough keyframes" rejection so the
|
|
1416
|
+
// user at least gets a discoverable
|
|
1417
|
+
// error rather than a silent hang.
|
|
1418
|
+
os_log(.fault, log: Self.diagLog,
|
|
1419
|
+
"[V16-batch-keyframe.fix7] single-keyframe copy failed: %{public}@",
|
|
1420
|
+
copyErr.localizedDescription)
|
|
1421
|
+
}
|
|
1422
|
+
}
|
|
1423
|
+
payload.collector?.cleanup()
|
|
1424
|
+
completion(nil, NSError(
|
|
1425
|
+
domain: "RNImageStitcherIncremental",
|
|
1426
|
+
code: 9003,
|
|
1427
|
+
userInfo: [NSLocalizedDescriptionKey:
|
|
1428
|
+
"Batch-keyframe finalize: 0 keyframes saved — capture didn't accept any frames."]
|
|
1429
|
+
))
|
|
1430
|
+
return
|
|
1431
|
+
}
|
|
1432
|
+
// Swift bridges `(NSError**)error` as `throws`,
|
|
1433
|
+
// so we use a do/catch instead of an inout error
|
|
1434
|
+
// pointer. Result is non-optional inside the
|
|
1435
|
+
// try block (or we wouldn't reach the success
|
|
1436
|
+
// branch).
|
|
1437
|
+
do {
|
|
1438
|
+
// V16 Phase 1.fix4 — ARCHITECTURAL PIVOT.
|
|
1439
|
+
// Three iterations of pose-driven (fix1/2/3)
|
|
1440
|
+
// produced different failure modes (UMat
|
|
1441
|
+
// bbox blowup, sideways output, frames out of
|
|
1442
|
+
// order, same physical object placed twice).
|
|
1443
|
+
// Per the systematic-debugging skill: 3+ failed
|
|
1444
|
+
// fixes on the same approach = wrong
|
|
1445
|
+
// architecture.
|
|
1446
|
+
//
|
|
1447
|
+
// Switching to cv::Stitcher's feature-matched
|
|
1448
|
+
// pipeline (stitchFramePaths) — same warper /
|
|
1449
|
+
// blender / seam settings, but uses ORB +
|
|
1450
|
+
// BFMatcher + RANSAC + BundleAdjusterRay +
|
|
1451
|
+
// waveCorrect for camera placement instead of
|
|
1452
|
+
// ARKit poses. Battle-tested for years in
|
|
1453
|
+
// cv::Stitcher::PANORAMA / SCANS modes.
|
|
1454
|
+
//
|
|
1455
|
+
// The 4-6 keyframes (with ≥40% new content
|
|
1456
|
+
// each, guaranteed by the Phase 0 gate) have
|
|
1457
|
+
// 60% overlap on retail content — features
|
|
1458
|
+
// are abundant and BA converges in <500 ms.
|
|
1459
|
+
// Output orientation, frame ordering, and
|
|
1460
|
+
// canvas bounds are all determined BY THE
|
|
1461
|
+
// FEATURES, not by pose-convention assumptions.
|
|
1462
|
+
//
|
|
1463
|
+
// ARKit poses are still saved alongside each
|
|
1464
|
+
// keyframe (`keyframePoses`) for future
|
|
1465
|
+
// pose-driven investigation as a separate
|
|
1466
|
+
// workstream — that path stays in the codebase
|
|
1467
|
+
// (stitchKeyframePaths method) but isn't on
|
|
1468
|
+
// the hot path.
|
|
1469
|
+
// V16 Phase 1b.fix3 — pass the EXIF Orientation
|
|
1470
|
+
// tag derived from `frameRotationDegrees`.
|
|
1471
|
+
// V16 Phase 1b.fix8 (C2) — read knobs from
|
|
1472
|
+
// `payload` (value-snapshot built under
|
|
1473
|
+
// stateLock in finalize's prologue). No
|
|
1474
|
+
// `self.*` access here; closure cannot race
|
|
1475
|
+
// with a concurrent `start()`.
|
|
1476
|
+
// AR-STITCHING-TWO-MODES — see memory/ar-stitching-two-modes.md
|
|
1477
|
+
// `payload.captureOrientation` drives the .mm
|
|
1478
|
+
// bake-rotation; `payload.keyframeExifOrientation`
|
|
1479
|
+
// kept for any future per-keyframe EXIF needs.
|
|
1480
|
+
_ = payload.keyframeExifOrientation
|
|
1481
|
+
os_log(.fault, log: Self.diagLog,
|
|
1482
|
+
"[V16-batch-keyframe] stitch (feature-matched): warper=%{public}@ blender=%{public}@ seam=%{public}@ captureOrientation=%{public}@",
|
|
1483
|
+
payload.batchWarperType,
|
|
1484
|
+
payload.batchBlenderType,
|
|
1485
|
+
payload.batchSeamFinderType,
|
|
1486
|
+
payload.captureOrientation)
|
|
1487
|
+
let r = try OpenCVStitcher.stitchFramePaths(
|
|
1488
|
+
payload.paths,
|
|
1489
|
+
outputPath: payload.cleaned,
|
|
1490
|
+
jpegQuality: payload.q,
|
|
1491
|
+
warperType: payload.batchWarperType,
|
|
1492
|
+
blenderType: payload.batchBlenderType,
|
|
1493
|
+
seamFinderType: payload.batchSeamFinderType,
|
|
1494
|
+
captureOrientation: payload.captureOrientation,
|
|
1495
|
+
useInscribedRectCrop: payload.batchEnableInscribedRectCrop
|
|
1496
|
+
)
|
|
1497
|
+
// V16 fix-attempt 9 (verified on device,
|
|
1498
|
+
// 2026-05-13) — sentinel-result detection.
|
|
1499
|
+
//
|
|
1500
|
+
// Background: 8 prior fix attempts (fix1-fix8)
|
|
1501
|
+
// chased a deterministic SEGV in Swift's
|
|
1502
|
+
// try-bridge over OpenCVStitcher's NSError-
|
|
1503
|
+
// out-parameter return. ASan with Sentry
|
|
1504
|
+
// disabled (.ips 172125) localised the SEGV to
|
|
1505
|
+
// an objc_retain on a wild pointer in unmapped
|
|
1506
|
+
// VM (0x60007a530, ReportDeadlySignal — no
|
|
1507
|
+
// shadow-memory match) immediately after the
|
|
1508
|
+
// .mm `return nil`. The fix-9 NULL TEST
|
|
1509
|
+
// (2026-05-13 17:21) replaced the two
|
|
1510
|
+
// immediate-repro `*error+return nil` sites
|
|
1511
|
+
// with non-nil sentinel returns; the crash
|
|
1512
|
+
// went away cleanly. After the test passed
|
|
1513
|
+
// we extended the sentinel pattern to ALL six
|
|
1514
|
+
// failure-return sites in stitchFramePaths
|
|
1515
|
+
// (pre-stitch memory abort, frames<2,
|
|
1516
|
+
// loadFramesOrFail, validPairs<1, workFrames<2,
|
|
1517
|
+
// estimator failure) so a production trigger of
|
|
1518
|
+
// any of them produces a clean error surface
|
|
1519
|
+
// instead of crashing the same way.
|
|
1520
|
+
//
|
|
1521
|
+
// We can't tell which underlying cause produced
|
|
1522
|
+
// a given sentinel from Swift (the .mm logs the
|
|
1523
|
+
// specific reason via NSLog so Console.app
|
|
1524
|
+
// shows it). JS gets one generic error code
|
|
1525
|
+
// (9007); refining the JS-facing taxonomy is
|
|
1526
|
+
// a follow-up if/when product needs differ-
|
|
1527
|
+
// entiated UX per cause.
|
|
1528
|
+
//
|
|
1529
|
+
// See: docs/site-content/design/2026-05-12-finalize-crash-investigation.md
|
|
1530
|
+
if r.width == 0 && r.height == 0 {
|
|
1531
|
+
os_log(.fault, log: Self.diagLog,
|
|
1532
|
+
"[V16-batch-keyframe.fix9] sentinel result from stitchFramePaths — see preceding [BatchStitcher] NSLog for cause; emitting clean error to JS")
|
|
1533
|
+
completion(nil, NSError(
|
|
1534
|
+
domain: "RNImageStitcherIncremental",
|
|
1535
|
+
code: 9007,
|
|
1536
|
+
userInfo: [NSLocalizedDescriptionKey:
|
|
1537
|
+
"Could not stitch the captured frames into a panorama. Please try recapturing with a slower, more deliberate pan that overlaps each section by at least 50%."]
|
|
1538
|
+
))
|
|
1539
|
+
return
|
|
1540
|
+
}
|
|
1541
|
+
// Keep saved keyframes on disk for post-hoc
|
|
1542
|
+
// re-processing (Ram's request). Cleanup is
|
|
1543
|
+
// a follow-up debug-menu task.
|
|
1544
|
+
// 2026-05-16 (Issue 5) — surface C+D
|
|
1545
|
+
// progressive-confidence retry telemetry to JS
|
|
1546
|
+
// so the host can render a debug toast. -1
|
|
1547
|
+
// sentinels = "no retry data" (early-return
|
|
1548
|
+
// success paths bypass the retry loop).
|
|
1549
|
+
var batchDict: [String: Any] = [
|
|
1550
|
+
"panoramaPath": r.outputPath,
|
|
1551
|
+
"width": Int(r.width),
|
|
1552
|
+
"height": Int(r.height),
|
|
1553
|
+
"acceptedCount": payload.paths.count,
|
|
1554
|
+
"droppedBackpressure": payload.drops,
|
|
1555
|
+
"batchKeyframeSessionDir":
|
|
1556
|
+
payload.collector?.sessionDir ?? "",
|
|
1557
|
+
"batchKeyframeCount": payload.paths.count,
|
|
1558
|
+
]
|
|
1559
|
+
if r.framesRequested >= 0 {
|
|
1560
|
+
batchDict["framesRequested"] = Int(r.framesRequested)
|
|
1561
|
+
}
|
|
1562
|
+
if r.framesIncluded >= 0 {
|
|
1563
|
+
batchDict["framesIncluded"] = Int(r.framesIncluded)
|
|
1564
|
+
if r.framesRequested >= 0 {
|
|
1565
|
+
batchDict["framesDropped"] =
|
|
1566
|
+
Int(r.framesRequested - r.framesIncluded)
|
|
1567
|
+
}
|
|
1568
|
+
}
|
|
1569
|
+
if r.finalConfidenceThresh >= 0 {
|
|
1570
|
+
batchDict["finalConfidenceThresh"] = r.finalConfidenceThresh
|
|
1571
|
+
}
|
|
1572
|
+
completion(batchDict, nil)
|
|
1573
|
+
} catch let stitchErr as NSError {
|
|
1574
|
+
completion(nil, stitchErr)
|
|
1575
|
+
}
|
|
1576
|
+
} else if let hybrid = payload.hybrid {
|
|
1577
|
+
let snap = try hybrid.finalize(atPath: payload.cleaned, jpegQuality: payload.q)
|
|
1578
|
+
completion([
|
|
1579
|
+
"panoramaPath": snap.panoramaPath,
|
|
1580
|
+
"width": snap.width,
|
|
1581
|
+
"height": snap.height,
|
|
1582
|
+
"acceptedCount": snap.acceptedCount,
|
|
1583
|
+
"droppedBackpressure": payload.drops,
|
|
1584
|
+
], nil)
|
|
1585
|
+
// 2026-05-16 — realtime+batch fusion (Option A
|
|
1586
|
+
// "Replace on completion") hook. The live
|
|
1587
|
+
// panorama has been written and the JS finalize
|
|
1588
|
+
// promise has resolved; now fire-and-forget an
|
|
1589
|
+
// async refinement over the hybrid engine's
|
|
1590
|
+
// accepted keyframes.
|
|
1591
|
+
//
|
|
1592
|
+
// Constraints honoured here (per the design doc
|
|
1593
|
+
// and the prompt's "Constraints" list):
|
|
1594
|
+
// 1. Hybrid realtime engine is NOT modified —
|
|
1595
|
+
// `OpenCVIncrementalStitcher.mm` stays
|
|
1596
|
+
// untouched; we only consult the existing
|
|
1597
|
+
// keyframe-path ivar that finalize() already
|
|
1598
|
+
// snapshotted into `payload.paths`.
|
|
1599
|
+
// 2. NO-OP when keyframes are not on disk.
|
|
1600
|
+
// Today's hybrid engine does NOT save per-
|
|
1601
|
+
// frame JPEGs (only batch-keyframe mode does
|
|
1602
|
+
// via OpenCVKeyframeCollector), so
|
|
1603
|
+
// `payload.paths` is empty for the hybrid
|
|
1604
|
+
// branch. `runHybridAutoRefine` detects
|
|
1605
|
+
// that and emits `isRefining=false` without
|
|
1606
|
+
// running cv::Stitcher. When a future change
|
|
1607
|
+
// hooks the hybrid engine up to a keyframe
|
|
1608
|
+
// collector, the same code path lights up
|
|
1609
|
+
// automatically.
|
|
1610
|
+
// 3. Refinement is fire-and-forget — finalize's
|
|
1611
|
+
// promise has ALREADY been resolved above.
|
|
1612
|
+
//
|
|
1613
|
+
// Capture-list discipline (C2 invariant — see the
|
|
1614
|
+
// file-top markers). No `self.*` references allowed
|
|
1615
|
+
// here; we route the dispatch through the type
|
|
1616
|
+
// (IncrementalStitcher.shared) so the
|
|
1617
|
+
// closure captures only value-typed locals + the
|
|
1618
|
+
// class type itself. shared is a process-wide
|
|
1619
|
+
// singleton (initialised once at module load),
|
|
1620
|
+
// so this is lifecycle-safe.
|
|
1621
|
+
let refinedOut = Self.refinedPathFromLive(
|
|
1622
|
+
livePath: snap.panoramaPath
|
|
1623
|
+
)
|
|
1624
|
+
let pathsForRefine = payload.paths // empty for hybrid today
|
|
1625
|
+
let capOri = payload.captureOrientation
|
|
1626
|
+
let warper = payload.batchWarperType
|
|
1627
|
+
let blender = payload.batchBlenderType
|
|
1628
|
+
let seam = payload.batchSeamFinderType
|
|
1629
|
+
let inscribed = payload.batchEnableInscribedRectCrop
|
|
1630
|
+
IncrementalStitcher.shared.refineQueue.async {
|
|
1631
|
+
IncrementalStitcher.shared.runHybridAutoRefine(
|
|
1632
|
+
framePaths: pathsForRefine,
|
|
1633
|
+
refinedOutputPath: refinedOut,
|
|
1634
|
+
captureOrientation: capOri,
|
|
1635
|
+
warperType: warper,
|
|
1636
|
+
blenderType: blender,
|
|
1637
|
+
seamFinderType: seam,
|
|
1638
|
+
useInscribedRectCrop: inscribed
|
|
1639
|
+
)
|
|
1640
|
+
}
|
|
1641
|
+
} else if let slit = payload.slit {
|
|
1642
|
+
let snap = try slit.finalize(atPath: payload.cleaned, jpegQuality: payload.q)
|
|
1643
|
+
completion([
|
|
1644
|
+
"panoramaPath": snap.panoramaPath,
|
|
1645
|
+
"width": snap.width,
|
|
1646
|
+
"height": snap.height,
|
|
1647
|
+
"acceptedCount": snap.acceptedCount,
|
|
1648
|
+
"droppedBackpressure": payload.drops,
|
|
1649
|
+
], nil)
|
|
1650
|
+
} else {
|
|
1651
|
+
completion(nil, NSError(
|
|
1652
|
+
domain: "RNImageStitcherIncremental",
|
|
1653
|
+
code: 9002,
|
|
1654
|
+
userInfo: [NSLocalizedDescriptionKey:
|
|
1655
|
+
"No active capture — call start() first."]
|
|
1656
|
+
))
|
|
1657
|
+
}
|
|
1658
|
+
} catch let err as NSError {
|
|
1659
|
+
completion(nil, err)
|
|
1660
|
+
}
|
|
1661
|
+
}
|
|
1662
|
+
// MARK: C2-INVARIANT-END
|
|
1663
|
+
}
|
|
1664
|
+
|
|
1665
|
+
/// 2026-05-16 — realtime+batch fusion (Option A) entry point.
|
|
1666
|
+
/// Runs the shared C++ stitcher over the supplied keyframe JPEGs
|
|
1667
|
+
/// and writes a refined panorama to `outputPath`.
|
|
1668
|
+
///
|
|
1669
|
+
/// Called by:
|
|
1670
|
+
/// 1. The bridge layer (explicit JS `refinePanorama(...)` API).
|
|
1671
|
+
/// 2. `runHybridAutoRefine(...)` below, the fire-and-forget hook
|
|
1672
|
+
/// from `finalize()` for the hybrid-engine path.
|
|
1673
|
+
///
|
|
1674
|
+
/// Threading: the work itself dispatches onto `refineQueue` (NOT
|
|
1675
|
+
/// `workQueue`). That keeps the per-capture path completely
|
|
1676
|
+
/// independent — a refinement in flight does not delay a fresh
|
|
1677
|
+
/// start()/consumeFrame() pair the operator may have initiated
|
|
1678
|
+
/// while the refinement runs. Completion fires on `refineQueue`;
|
|
1679
|
+
/// callers that need main-thread delivery (e.g. the bridge
|
|
1680
|
+
/// promise resolver) re-dispatch as needed.
|
|
1681
|
+
///
|
|
1682
|
+
/// Configuration: same option set the bridge sees from JS plus
|
|
1683
|
+
/// production-tested defaults that match the existing
|
|
1684
|
+
/// batch-keyframe finalize path:
|
|
1685
|
+
/// warperType = "spherical"
|
|
1686
|
+
/// blenderType = "multiband"
|
|
1687
|
+
/// seamFinderType = "graphcut"
|
|
1688
|
+
/// captureOrientation = "portrait"
|
|
1689
|
+
/// useInscribedRectCrop = false
|
|
1690
|
+
/// jpegQuality = 90
|
|
1691
|
+
///
|
|
1692
|
+
/// Pre-conditions enforced here (in addition to bridge-level
|
|
1693
|
+
/// validation): every input path must exist on disk; if any is
|
|
1694
|
+
/// missing the call resolves with an NSError so the caller can
|
|
1695
|
+
/// surface a clean error rather than letting cv::imread crash
|
|
1696
|
+
/// inside the manual pipeline.
|
|
1697
|
+
@objc public func refinePanorama(
|
|
1698
|
+
framePaths: [String],
|
|
1699
|
+
outputPath: String,
|
|
1700
|
+
config: [String: Any],
|
|
1701
|
+
completion: @escaping ([String: Any]?, NSError?) -> Void
|
|
1702
|
+
) {
|
|
1703
|
+
guard framePaths.count >= 2 else {
|
|
1704
|
+
completion(nil, NSError(
|
|
1705
|
+
domain: "RNImageStitcherIncremental",
|
|
1706
|
+
code: 9101,
|
|
1707
|
+
userInfo: [NSLocalizedDescriptionKey:
|
|
1708
|
+
"refinePanorama requires at least 2 framePaths (got \(framePaths.count))."]
|
|
1709
|
+
))
|
|
1710
|
+
return
|
|
1711
|
+
}
|
|
1712
|
+
let fm = FileManager.default
|
|
1713
|
+
for p in framePaths {
|
|
1714
|
+
let cleaned = p.hasPrefix("file://") ? String(p.dropFirst(7)) : p
|
|
1715
|
+
if !fm.fileExists(atPath: cleaned) {
|
|
1716
|
+
completion(nil, NSError(
|
|
1717
|
+
domain: "RNImageStitcherIncremental",
|
|
1718
|
+
code: 9102,
|
|
1719
|
+
userInfo: [NSLocalizedDescriptionKey:
|
|
1720
|
+
"refinePanorama: keyframe missing on disk — \(cleaned)"]
|
|
1721
|
+
))
|
|
1722
|
+
return
|
|
1723
|
+
}
|
|
1724
|
+
}
|
|
1725
|
+
let warper = (config["warperType"] as? String) ?? "spherical"
|
|
1726
|
+
let blender = (config["blenderType"] as? String) ?? "multiband"
|
|
1727
|
+
let seam = (config["seamFinderType"] as? String) ?? "graphcut"
|
|
1728
|
+
let orientation = (config["captureOrientation"] as? String) ?? "portrait"
|
|
1729
|
+
let useInscribed = (config["useInscribedRectCrop"] as? Bool) ?? false
|
|
1730
|
+
let quality = max(1, min(100, (config["jpegQuality"] as? Int) ?? 90))
|
|
1731
|
+
let cleanedOutput = outputPath.hasPrefix("file://")
|
|
1732
|
+
? String(outputPath.dropFirst(7))
|
|
1733
|
+
: outputPath
|
|
1734
|
+
|
|
1735
|
+
os_log(.fault, log: Self.diagLog,
|
|
1736
|
+
"[refine] dispatch frames=%d output=%{public}@ warper=%{public}@ blender=%{public}@ seam=%{public}@",
|
|
1737
|
+
framePaths.count,
|
|
1738
|
+
cleanedOutput,
|
|
1739
|
+
warper, blender, seam)
|
|
1740
|
+
|
|
1741
|
+
refineQueue.async {
|
|
1742
|
+
// C2-style: closure captures only value-typed locals
|
|
1743
|
+
// (paths, output path, config strings). No `self` access
|
|
1744
|
+
// is needed for the cv::Stitcher call — OpenCVStitcher is
|
|
1745
|
+
// a class method, not an instance method, so we can call
|
|
1746
|
+
// it directly via the type.
|
|
1747
|
+
do {
|
|
1748
|
+
let r = try OpenCVStitcher.stitchFramePaths(
|
|
1749
|
+
framePaths,
|
|
1750
|
+
outputPath: cleanedOutput,
|
|
1751
|
+
jpegQuality: quality,
|
|
1752
|
+
warperType: warper,
|
|
1753
|
+
blenderType: blender,
|
|
1754
|
+
seamFinderType: seam,
|
|
1755
|
+
captureOrientation: orientation,
|
|
1756
|
+
useInscribedRectCrop: useInscribed
|
|
1757
|
+
)
|
|
1758
|
+
// fix-9 sentinel detection — see the finalize() path
|
|
1759
|
+
// for the full rationale. A 0×0 result means
|
|
1760
|
+
// OpenCVStitcher hit one of its six guarded failure
|
|
1761
|
+
// returns; surface as a clean NSError.
|
|
1762
|
+
if r.width == 0 && r.height == 0 {
|
|
1763
|
+
completion(nil, NSError(
|
|
1764
|
+
domain: "RNImageStitcherIncremental",
|
|
1765
|
+
code: 9107,
|
|
1766
|
+
userInfo: [NSLocalizedDescriptionKey:
|
|
1767
|
+
"refinePanorama: stitcher returned sentinel — see preceding [BatchStitcher] log for cause."]
|
|
1768
|
+
))
|
|
1769
|
+
return
|
|
1770
|
+
}
|
|
1771
|
+
completion([
|
|
1772
|
+
"panoramaPath": r.outputPath,
|
|
1773
|
+
"width": Int(r.width),
|
|
1774
|
+
"height": Int(r.height),
|
|
1775
|
+
"framesRequested": framePaths.count,
|
|
1776
|
+
"framesIncluded": framePaths.count,
|
|
1777
|
+
"framesDropped": 0,
|
|
1778
|
+
"finalConfidenceThresh": -1.0,
|
|
1779
|
+
], nil)
|
|
1780
|
+
} catch let err as NSError {
|
|
1781
|
+
completion(nil, err)
|
|
1782
|
+
}
|
|
1783
|
+
}
|
|
1784
|
+
}
|
|
1785
|
+
|
|
1786
|
+
/// 2026-05-16 — realtime+batch fusion (Option A) auto-trigger.
|
|
1787
|
+
/// Called from `finalize()` immediately after the hybrid engine
|
|
1788
|
+
/// wrote its live panorama; fire-and-forget from finalize()'s
|
|
1789
|
+
/// perspective so the JS-side finalize promise resolves with the
|
|
1790
|
+
/// live result first. Then this method:
|
|
1791
|
+
///
|
|
1792
|
+
/// 1. Emits a state event with `isRefining = true` so the host
|
|
1793
|
+
/// can render its "Refining…" pill.
|
|
1794
|
+
/// 2. Runs `refinePanorama(framePaths, refinedOutputPath, ...)`.
|
|
1795
|
+
/// 3. On success: emits a state event with `isRefining = false`
|
|
1796
|
+
/// AND `refinedPanoramaPath = <path>` so the host swaps in
|
|
1797
|
+
/// the higher-quality output.
|
|
1798
|
+
/// 4. On failure: emits a state event with `isRefining = false`
|
|
1799
|
+
/// AND NO refined path. Host keeps showing the live
|
|
1800
|
+
/// panorama; the design doc's "Couldn't refine" toast UX is
|
|
1801
|
+
/// a follow-up.
|
|
1802
|
+
///
|
|
1803
|
+
/// No-op when `framePaths.count < 2` or any framePath is missing
|
|
1804
|
+
/// on disk. Hybrid-engine captures DO NOT today save per-frame
|
|
1805
|
+
/// JPEGs, so this method's most common call site (from finalize's
|
|
1806
|
+
/// hybrid branch) currently produces a no-op + isRefining=false
|
|
1807
|
+
/// emit — which is intentional (the design doc says "if
|
|
1808
|
+
/// keyframes are NOT on disk, the auto-trigger is a no-op").
|
|
1809
|
+
private func runHybridAutoRefine(
|
|
1810
|
+
framePaths: [String],
|
|
1811
|
+
refinedOutputPath: String,
|
|
1812
|
+
captureOrientation: String,
|
|
1813
|
+
warperType: String,
|
|
1814
|
+
blenderType: String,
|
|
1815
|
+
seamFinderType: String,
|
|
1816
|
+
useInscribedRectCrop: Bool
|
|
1817
|
+
) {
|
|
1818
|
+
if framePaths.count < 2 {
|
|
1819
|
+
os_log(.info, log: Self.diagLog,
|
|
1820
|
+
"[refine.auto] skipped: framePaths.count=%d (< 2 — hybrid engine retains no per-frame JPEGs)",
|
|
1821
|
+
framePaths.count)
|
|
1822
|
+
// Emit isRefining=false so any host that pre-seeded a
|
|
1823
|
+
// pill on finalize doesn't get stuck.
|
|
1824
|
+
self.emitRefinementState(isRefining: false, refinedPanoramaPath: nil)
|
|
1825
|
+
return
|
|
1826
|
+
}
|
|
1827
|
+
// Pre-flight existence check so we degrade gracefully when
|
|
1828
|
+
// a JPEG was unlinked between finalize and the dispatch
|
|
1829
|
+
// landing on refineQueue.
|
|
1830
|
+
let fm = FileManager.default
|
|
1831
|
+
for p in framePaths {
|
|
1832
|
+
let cleaned = p.hasPrefix("file://") ? String(p.dropFirst(7)) : p
|
|
1833
|
+
if !fm.fileExists(atPath: cleaned) {
|
|
1834
|
+
os_log(.info, log: Self.diagLog,
|
|
1835
|
+
"[refine.auto] skipped: missing keyframe %{public}@",
|
|
1836
|
+
cleaned)
|
|
1837
|
+
self.emitRefinementState(isRefining: false, refinedPanoramaPath: nil)
|
|
1838
|
+
return
|
|
1839
|
+
}
|
|
1840
|
+
}
|
|
1841
|
+
// Signal the pill on before the stitcher work begins. The
|
|
1842
|
+
// emit goes through the same notification channel as every
|
|
1843
|
+
// other state update; JS sees it asynchronously, which is
|
|
1844
|
+
// fine — operator UX wants the pill within a few hundred ms,
|
|
1845
|
+
// not synchronously with finalize's promise resolution.
|
|
1846
|
+
self.emitRefinementState(isRefining: true, refinedPanoramaPath: nil)
|
|
1847
|
+
let config: [String: Any] = [
|
|
1848
|
+
"warperType": warperType,
|
|
1849
|
+
"blenderType": blenderType,
|
|
1850
|
+
"seamFinderType": seamFinderType,
|
|
1851
|
+
"captureOrientation": captureOrientation,
|
|
1852
|
+
"useInscribedRectCrop": useInscribedRectCrop,
|
|
1853
|
+
"jpegQuality": 90,
|
|
1854
|
+
]
|
|
1855
|
+
self.refinePanorama(
|
|
1856
|
+
framePaths: framePaths,
|
|
1857
|
+
outputPath: refinedOutputPath,
|
|
1858
|
+
config: config
|
|
1859
|
+
) { [weak self] result, error in
|
|
1860
|
+
guard let self = self else { return }
|
|
1861
|
+
if let error = error {
|
|
1862
|
+
os_log(.fault, log: Self.diagLog,
|
|
1863
|
+
"[refine.auto] refinement failed: %{public}@ — leaving live output in place",
|
|
1864
|
+
error.localizedDescription)
|
|
1865
|
+
self.emitRefinementState(isRefining: false, refinedPanoramaPath: nil)
|
|
1866
|
+
return
|
|
1867
|
+
}
|
|
1868
|
+
let path = (result?["panoramaPath"] as? String) ?? refinedOutputPath
|
|
1869
|
+
os_log(.fault, log: Self.diagLog,
|
|
1870
|
+
"[refine.auto] success path=%{public}@",
|
|
1871
|
+
path)
|
|
1872
|
+
self.emitRefinementState(isRefining: false, refinedPanoramaPath: path)
|
|
1873
|
+
}
|
|
1874
|
+
}
|
|
1875
|
+
|
|
1876
|
+
/// 2026-05-16 — emit a minimal state event carrying only the
|
|
1877
|
+
/// refinement-related fields. Mirrors the existing
|
|
1878
|
+
/// `emitBatchKeyframeAcceptedState` pattern: build a fresh
|
|
1879
|
+
/// IncrementalStateObject, then add the new optional fields
|
|
1880
|
+
/// directly to the userInfo dict so JS (which reads from the
|
|
1881
|
+
/// raw event payload) picks them up without a schema change in
|
|
1882
|
+
/// the Obj-C class.
|
|
1883
|
+
private func emitRefinementState(
|
|
1884
|
+
isRefining: Bool,
|
|
1885
|
+
refinedPanoramaPath: String?
|
|
1886
|
+
) {
|
|
1887
|
+
// Preserve the most-recent panoramaPath / dims / accepted
|
|
1888
|
+
// count so the JS subscriber's sticky-snapshot merge keeps
|
|
1889
|
+
// showing the live preview between the finalize and the
|
|
1890
|
+
// refined swap. All other fields default to "no-op" values.
|
|
1891
|
+
stateLock.lock()
|
|
1892
|
+
let prev = self.lastState
|
|
1893
|
+
stateLock.unlock()
|
|
1894
|
+
let state = IncrementalStateObject(
|
|
1895
|
+
panoramaPath: prev?.panoramaPath,
|
|
1896
|
+
width: prev?.width ?? 0,
|
|
1897
|
+
height: prev?.height ?? 0,
|
|
1898
|
+
acceptedCount: prev?.acceptedCount ?? 0,
|
|
1899
|
+
outcome: prev?.outcome ?? .acceptedHigh,
|
|
1900
|
+
confidence: prev?.confidence ?? 1.0,
|
|
1901
|
+
overlapPercent: prev?.overlapPercent ?? -1.0,
|
|
1902
|
+
processingMs: 0,
|
|
1903
|
+
isLandscape: prev?.isLandscape ?? false,
|
|
1904
|
+
paintedExtent: prev?.paintedExtent ?? 0,
|
|
1905
|
+
panExtent: prev?.panExtent ?? 0,
|
|
1906
|
+
keyframeMax: prev?.keyframeMax ?? 0
|
|
1907
|
+
)
|
|
1908
|
+
var dict = state.asDictionary()
|
|
1909
|
+
dict["isRefining"] = isRefining
|
|
1910
|
+
if let p = refinedPanoramaPath {
|
|
1911
|
+
dict["refinedPanoramaPath"] = p
|
|
1912
|
+
}
|
|
1913
|
+
NotificationCenter.default.post(
|
|
1914
|
+
name: .retailensIncrementalStateUpdate,
|
|
1915
|
+
object: nil,
|
|
1916
|
+
userInfo: dict
|
|
1917
|
+
)
|
|
1918
|
+
}
|
|
1919
|
+
|
|
1920
|
+
/// Cancel an in-progress capture without producing output.
|
|
1921
|
+
/// Same V12.1 synchronous-stop pattern as finalize.
|
|
1922
|
+
@objc public func cancel() {
|
|
1923
|
+
// V12.9 fix #3 — same ordering as finalize: flip isRunning
|
|
1924
|
+
// FIRST so any in-flight consumeFrame bails at its first
|
|
1925
|
+
// guard. Then detach the AR consumer.
|
|
1926
|
+
stateLock.lock()
|
|
1927
|
+
let hybrid = self.hybridEngine
|
|
1928
|
+
let slit = self.firstwinsEngine
|
|
1929
|
+
let collector = self.keyframeCollector
|
|
1930
|
+
self.hybridEngine = nil
|
|
1931
|
+
self.firstwinsEngine = nil
|
|
1932
|
+
self.keyframeCollector = nil
|
|
1933
|
+
self.batchKeyframeMode = false
|
|
1934
|
+
self.keyframePaths = []
|
|
1935
|
+
self.keyframePoses = []
|
|
1936
|
+
self.isRunning = false
|
|
1937
|
+
self.lastState = nil
|
|
1938
|
+
// V16 — reset the keyframe gate so the next start() begins
|
|
1939
|
+
// with a clean polygon state and counter. Safe to do under
|
|
1940
|
+
// stateLock because the gate is only mutated from the AR
|
|
1941
|
+
// delegate (consumeFrame) and the JS thread (start/cancel
|
|
1942
|
+
// /markNextFrameAsLastKeyframe), all serialized via this lock.
|
|
1943
|
+
self.keyframeGate.reset()
|
|
1944
|
+
stateLock.unlock()
|
|
1945
|
+
RNSARSession.shared.incrementalConsumer = nil
|
|
1946
|
+
// Reset on the work queue so we don't race with an in-flight
|
|
1947
|
+
// ingest that's still touching the engine's canvas. Cancel
|
|
1948
|
+
// ALSO removes the collector's session directory — the
|
|
1949
|
+
// operator explicitly aborted, so the saved JPEGs aren't
|
|
1950
|
+
// worth keeping for re-processing.
|
|
1951
|
+
workQueue.async {
|
|
1952
|
+
hybrid?.reset()
|
|
1953
|
+
slit?.reset()
|
|
1954
|
+
collector?.cleanup()
|
|
1955
|
+
}
|
|
1956
|
+
}
|
|
1957
|
+
|
|
1958
|
+
/// V16 — JS-side hook for shutter-release: arm the gate so the
|
|
1959
|
+
/// NEXT delivered ARFrame is force-accepted regardless of overlap.
|
|
1960
|
+
/// Without this, the user releasing mid-pan (between two natural
|
|
1961
|
+
/// keyframe boundaries) would leave the trailing edge of the
|
|
1962
|
+
/// scene unrepresented in the panorama.
|
|
1963
|
+
///
|
|
1964
|
+
/// Idempotent: setting the flag while it's already set is a no-op.
|
|
1965
|
+
/// Safe to call from any thread (NSLock-guarded). No-op when the
|
|
1966
|
+
/// gate is disabled (frameSelectionMode = "time-based").
|
|
1967
|
+
@objc public func markNextFrameAsLastKeyframe() {
|
|
1968
|
+
stateLock.lock()
|
|
1969
|
+
defer { stateLock.unlock() }
|
|
1970
|
+
guard self.isRunning, self.keyframeGate.enabled else { return }
|
|
1971
|
+
self.keyframeGate.forceAcceptNext = true
|
|
1972
|
+
os_log(.fault, log: Self.diagLog,
|
|
1973
|
+
"[V16-keyframe] markNextFrameAsLastKeyframe armed (count=%d max=%d)",
|
|
1974
|
+
self.keyframeGate.acceptedCount, self.keyframeGate.maxCount)
|
|
1975
|
+
}
|
|
1976
|
+
|
|
1977
|
+
/// Whether the engine is currently in batch-keyframe mode.
|
|
1978
|
+
/// Bridge reads this to decide whether the JS-driven
|
|
1979
|
+
/// `processFrameAtPath` path can use the lightweight
|
|
1980
|
+
/// `addBatchKeyframePath` (path-only) entry below.
|
|
1981
|
+
@objc public var isBatchKeyframeMode: Bool {
|
|
1982
|
+
stateLock.lock()
|
|
1983
|
+
defer { stateLock.unlock() }
|
|
1984
|
+
return batchKeyframeMode
|
|
1985
|
+
}
|
|
1986
|
+
|
|
1987
|
+
/// 2026-05-18 (Issue #2 v2) — JS-driver entry-point for
|
|
1988
|
+
/// batch-keyframe captures. Mirrors Android's behaviour: the
|
|
1989
|
+
/// caller (JS side via the IncrementalStitcherBridge) hands us a
|
|
1990
|
+
/// JPEG file path that already exists on disk (saved by
|
|
1991
|
+
/// vision-camera's takeSnapshot), plus a synthetic pose derived
|
|
1992
|
+
/// from gyro integration. We:
|
|
1993
|
+
///
|
|
1994
|
+
/// 1. Validate state (running + batchKeyframeMode).
|
|
1995
|
+
/// 2. Ask the shared C++ KeyframeGate whether to accept this
|
|
1996
|
+
/// frame. Pass `latchedPlane: nil` — non-AR captures have
|
|
1997
|
+
/// no plane; the C++ gate falls back to a pose-only
|
|
1998
|
+
/// angular-delta strategy. We do NOT pass a pixel buffer:
|
|
1999
|
+
/// the Pose strategy doesn't need one, and avoiding the
|
|
2000
|
+
/// JPEG → CVPixelBuffer round-trip dodges the iOS
|
|
2001
|
+
/// orientation bugs that broke Issue 2 v1
|
|
2002
|
+
/// (UIImage/CGContext Y-flip + EXIF-vs-CGImage dimension
|
|
2003
|
+
/// mismatch — see the symptom in 2026-05-18 user report).
|
|
2004
|
+
/// 3. If accepted, append the existing path + pose to the
|
|
2005
|
+
/// finalize-time lists. No JPEG re-encode — the file on
|
|
2006
|
+
/// disk IS the keyframe. `retailens::stitchFramePaths()`
|
|
2007
|
+
/// at finalize uses `cv::imread` which natively handles
|
|
2008
|
+
/// EXIF orientation, so the output panorama reads upright.
|
|
2009
|
+
/// 4. Emit the same state-event the AR delegate path emits so
|
|
2010
|
+
/// the JS live band populates identically.
|
|
2011
|
+
///
|
|
2012
|
+
/// Architecture note: this is structurally parallel to Android's
|
|
2013
|
+
/// `IncrementalStitcher.kt::processFrameAtPath`
|
|
2014
|
+
/// `batchKeyframeMode` branch (lines 573-627). A follow-up
|
|
2015
|
+
/// should extract the dispatch (gate-eval + path-append + emit)
|
|
2016
|
+
/// into shared cpp/ so both platforms become 5-line wrappers
|
|
2017
|
+
/// around a single C++ entry point.
|
|
2018
|
+
@objc public func addBatchKeyframePath(
|
|
2019
|
+
path: String,
|
|
2020
|
+
pose: RNSARFramePose
|
|
2021
|
+
) -> Bool {
|
|
2022
|
+
stateLock.lock()
|
|
2023
|
+
guard self.isRunning, self.batchKeyframeMode else {
|
|
2024
|
+
stateLock.unlock()
|
|
2025
|
+
return false
|
|
2026
|
+
}
|
|
2027
|
+
stateLock.unlock()
|
|
2028
|
+
|
|
2029
|
+
// Pose-only gate evaluation — no pixel buffer, no plane.
|
|
2030
|
+
let decision = self.keyframeGate.evaluate(
|
|
2031
|
+
pose: pose,
|
|
2032
|
+
latchedPlane: nil
|
|
2033
|
+
)
|
|
2034
|
+
if !decision.accept {
|
|
2035
|
+
self.emitKeyframeRejectState(decision: decision)
|
|
2036
|
+
return false
|
|
2037
|
+
}
|
|
2038
|
+
|
|
2039
|
+
// Append path + pose to the finalize lists. Take the lock
|
|
2040
|
+
// briefly — these mutate state read by `finalize()`.
|
|
2041
|
+
stateLock.lock()
|
|
2042
|
+
self.keyframePaths.append(path)
|
|
2043
|
+
self.keyframePoses.append(pose.asDictionary())
|
|
2044
|
+
let count = self.keyframePaths.count
|
|
2045
|
+
stateLock.unlock()
|
|
2046
|
+
os_log(.fault, log: Self.diagLog,
|
|
2047
|
+
"[V16-batch-keyframe.js] accepted path #%d → %{public}@",
|
|
2048
|
+
Int32(count), path)
|
|
2049
|
+
self.emitBatchKeyframeAcceptedState(
|
|
2050
|
+
thumbnailPath: path,
|
|
2051
|
+
keyframeIndex: count - 1,
|
|
2052
|
+
keyframeCount: count,
|
|
2053
|
+
keyframeMax: self.keyframeGate.maxCount,
|
|
2054
|
+
isLandscape: pose.imageWidth >= pose.imageHeight
|
|
2055
|
+
)
|
|
2056
|
+
return true
|
|
2057
|
+
}
|
|
2058
|
+
|
|
2059
|
+
/// V16 Phase 1 — emit a state event when a batch-keyframe is
|
|
2060
|
+
/// saved. Carries the on-disk thumbnail path so JS can render it
|
|
2061
|
+
/// in LiveFrameStrip + advance the "Keyframes: N/M" pill.
|
|
2062
|
+
private func emitBatchKeyframeAcceptedState(
|
|
2063
|
+
thumbnailPath: String,
|
|
2064
|
+
keyframeIndex: Int,
|
|
2065
|
+
keyframeCount: Int,
|
|
2066
|
+
keyframeMax: Int,
|
|
2067
|
+
isLandscape: Bool
|
|
2068
|
+
) {
|
|
2069
|
+
let state = IncrementalStateObject(
|
|
2070
|
+
panoramaPath: nil,
|
|
2071
|
+
width: 0,
|
|
2072
|
+
height: 0,
|
|
2073
|
+
acceptedCount: keyframeCount,
|
|
2074
|
+
outcome: .acceptedHigh,
|
|
2075
|
+
confidence: 1.0,
|
|
2076
|
+
overlapPercent: -1.0,
|
|
2077
|
+
processingMs: 0,
|
|
2078
|
+
isLandscape: isLandscape,
|
|
2079
|
+
paintedExtent: 0,
|
|
2080
|
+
panExtent: 0,
|
|
2081
|
+
keyframeMax: keyframeMax
|
|
2082
|
+
)
|
|
2083
|
+
stateLock.lock()
|
|
2084
|
+
self.lastState = state
|
|
2085
|
+
stateLock.unlock()
|
|
2086
|
+
var dict = state.asDictionary()
|
|
2087
|
+
// Extra fields the existing IncrementalState schema doesn't
|
|
2088
|
+
// carry — JS reads these directly from the userInfo blob.
|
|
2089
|
+
dict["batchKeyframeThumbnailPath"] = thumbnailPath
|
|
2090
|
+
dict["batchKeyframeIndex"] = keyframeIndex
|
|
2091
|
+
NotificationCenter.default.post(
|
|
2092
|
+
name: .retailensIncrementalStateUpdate,
|
|
2093
|
+
object: nil,
|
|
2094
|
+
userInfo: dict
|
|
2095
|
+
)
|
|
2096
|
+
}
|
|
2097
|
+
|
|
2098
|
+
/// V16 Phase 1b.fix2 — deep-copy a CVPixelBuffer so it survives
|
|
2099
|
+
/// past ARKit's pool reuse window. Apple's contract is that
|
|
2100
|
+
/// ARFrame.capturedImage is only valid inside the delegate
|
|
2101
|
+
/// scope; CFRetain alone doesn't extend the underlying
|
|
2102
|
+
/// IOSurface's lifetime. This is the documented fix.
|
|
2103
|
+
///
|
|
2104
|
+
/// Handles both planar (NV12 — ARKit default) and packed
|
|
2105
|
+
/// (BGRA) formats. Returns nil on allocation failure.
|
|
2106
|
+
private static func deepCopyPixelBuffer(
|
|
2107
|
+
_ src: CVPixelBuffer
|
|
2108
|
+
) -> CVPixelBuffer? {
|
|
2109
|
+
let format = CVPixelBufferGetPixelFormatType(src)
|
|
2110
|
+
let width = CVPixelBufferGetWidth(src)
|
|
2111
|
+
let height = CVPixelBufferGetHeight(src)
|
|
2112
|
+
|
|
2113
|
+
// IOSurface-backed copy so cv::imread / Vision frameworks
|
|
2114
|
+
// can read the buffer without re-uploading. Empty dict =
|
|
2115
|
+
// use default IOSurface attributes.
|
|
2116
|
+
let attrs: NSDictionary = [
|
|
2117
|
+
kCVPixelBufferIOSurfacePropertiesKey: NSDictionary(),
|
|
2118
|
+
]
|
|
2119
|
+
var dst: CVPixelBuffer?
|
|
2120
|
+
let createStatus = CVPixelBufferCreate(
|
|
2121
|
+
kCFAllocatorDefault, width, height, format, attrs, &dst
|
|
2122
|
+
)
|
|
2123
|
+
guard createStatus == kCVReturnSuccess, let copy = dst else {
|
|
2124
|
+
return nil
|
|
2125
|
+
}
|
|
2126
|
+
|
|
2127
|
+
let srcLock = CVPixelBufferLockBaseAddress(src, .readOnly)
|
|
2128
|
+
let dstLock = CVPixelBufferLockBaseAddress(copy, [])
|
|
2129
|
+
defer {
|
|
2130
|
+
CVPixelBufferUnlockBaseAddress(src, .readOnly)
|
|
2131
|
+
CVPixelBufferUnlockBaseAddress(copy, [])
|
|
2132
|
+
}
|
|
2133
|
+
guard srcLock == kCVReturnSuccess,
|
|
2134
|
+
dstLock == kCVReturnSuccess else {
|
|
2135
|
+
return nil
|
|
2136
|
+
}
|
|
2137
|
+
|
|
2138
|
+
if CVPixelBufferIsPlanar(src) {
|
|
2139
|
+
let nPlanes = CVPixelBufferGetPlaneCount(src)
|
|
2140
|
+
for plane in 0..<nPlanes {
|
|
2141
|
+
guard
|
|
2142
|
+
let srcBase =
|
|
2143
|
+
CVPixelBufferGetBaseAddressOfPlane(src, plane),
|
|
2144
|
+
let dstBase =
|
|
2145
|
+
CVPixelBufferGetBaseAddressOfPlane(copy, plane)
|
|
2146
|
+
else { return nil }
|
|
2147
|
+
let srcStride =
|
|
2148
|
+
CVPixelBufferGetBytesPerRowOfPlane(src, plane)
|
|
2149
|
+
let dstStride =
|
|
2150
|
+
CVPixelBufferGetBytesPerRowOfPlane(copy, plane)
|
|
2151
|
+
let planeH =
|
|
2152
|
+
CVPixelBufferGetHeightOfPlane(src, plane)
|
|
2153
|
+
if srcStride == dstStride {
|
|
2154
|
+
memcpy(dstBase, srcBase, srcStride * planeH)
|
|
2155
|
+
} else {
|
|
2156
|
+
let rowBytes = min(srcStride, dstStride)
|
|
2157
|
+
for r in 0..<planeH {
|
|
2158
|
+
memcpy(
|
|
2159
|
+
dstBase.advanced(by: r * dstStride),
|
|
2160
|
+
srcBase.advanced(by: r * srcStride),
|
|
2161
|
+
rowBytes
|
|
2162
|
+
)
|
|
2163
|
+
}
|
|
2164
|
+
}
|
|
2165
|
+
}
|
|
2166
|
+
} else {
|
|
2167
|
+
// Packed (e.g. BGRA).
|
|
2168
|
+
guard let srcBase = CVPixelBufferGetBaseAddress(src),
|
|
2169
|
+
let dstBase = CVPixelBufferGetBaseAddress(copy)
|
|
2170
|
+
else { return nil }
|
|
2171
|
+
let srcStride = CVPixelBufferGetBytesPerRow(src)
|
|
2172
|
+
let dstStride = CVPixelBufferGetBytesPerRow(copy)
|
|
2173
|
+
if srcStride == dstStride {
|
|
2174
|
+
memcpy(dstBase, srcBase, srcStride * height)
|
|
2175
|
+
} else {
|
|
2176
|
+
let rowBytes = min(srcStride, dstStride)
|
|
2177
|
+
for r in 0..<height {
|
|
2178
|
+
memcpy(
|
|
2179
|
+
dstBase.advanced(by: r * dstStride),
|
|
2180
|
+
srcBase.advanced(by: r * srcStride),
|
|
2181
|
+
rowBytes
|
|
2182
|
+
)
|
|
2183
|
+
}
|
|
2184
|
+
}
|
|
2185
|
+
}
|
|
2186
|
+
return copy
|
|
2187
|
+
}
|
|
2188
|
+
|
|
2189
|
+
/// Synthesise + emit a state event for a frame the keyframe gate
|
|
2190
|
+
/// rejected. The native engine never sees the frame, so its own
|
|
2191
|
+
/// state machinery isn't invoked — but JS still wants the event
|
|
2192
|
+
/// so the status pill can update ("frame skipped, still 3/6").
|
|
2193
|
+
private func emitKeyframeRejectState(decision: KeyframeGateDecision) {
|
|
2194
|
+
// Pick the right outcome value for JS; defaults match the
|
|
2195
|
+
// intent (overlap-too-high vs cap-reached).
|
|
2196
|
+
let outcome: IncrementalOutcome
|
|
2197
|
+
switch decision.reason {
|
|
2198
|
+
case "max-reached": outcome = .skippedKeyframeMaxReached
|
|
2199
|
+
case "overlap-too-high": outcome = .skippedKeyframeOverlap
|
|
2200
|
+
default: outcome = .skippedKeyframeOverlap
|
|
2201
|
+
}
|
|
2202
|
+
// Re-use the previous state's pan-extent / orientation fields
|
|
2203
|
+
// so the band overlay doesn't flicker when a reject lands.
|
|
2204
|
+
//
|
|
2205
|
+
// V16 A2 (post-2026-05-13-14:41:57 .ips):
|
|
2206
|
+
// `self.lastState` is written from BOTH this method (main
|
|
2207
|
+
// thread, reject path) AND the engine accept-path
|
|
2208
|
+
// (workQueue, ~line 1908) — both under stateLock at the
|
|
2209
|
+
// write site, but these reads were unprotected. Pre-A2
|
|
2210
|
+
// the reject rate was a few per second and the race was
|
|
2211
|
+
// latent; A2's flow-based default raised it to ~50/s. At
|
|
2212
|
+
// that frequency the torn-pointer-on-class-ref race fires:
|
|
2213
|
+
// T0 workQueue prepares new state, about to replace
|
|
2214
|
+
// self.lastState
|
|
2215
|
+
// T1 main thread loads self.lastState — sees OLD ref
|
|
2216
|
+
// T2 workQueue writes new ref; ARC releases OLD
|
|
2217
|
+
// (refcount → 0 → freed)
|
|
2218
|
+
// T3 main thread's load completes; ARC tries to retain
|
|
2219
|
+
// OLD ref → objc_retain on freed memory →
|
|
2220
|
+
// EXC_BAD_ACCESS at frame 0 of emitKeyframeRejectState.
|
|
2221
|
+
// Fix: hold stateLock for the read. Cheap (microseconds);
|
|
2222
|
+
// tighter scope than wrapping the whole function so we
|
|
2223
|
+
// don't hold during the NotificationCenter post that
|
|
2224
|
+
// follows.
|
|
2225
|
+
let prev: IncrementalStateObject?
|
|
2226
|
+
let acceptedCount: Int
|
|
2227
|
+
stateLock.lock()
|
|
2228
|
+
prev = self.lastState
|
|
2229
|
+
acceptedCount = self.engineAcceptedCount
|
|
2230
|
+
stateLock.unlock()
|
|
2231
|
+
let overlapPercent = (decision.newContentFraction >= 0)
|
|
2232
|
+
? (1.0 - decision.newContentFraction) * 100.0
|
|
2233
|
+
: (prev?.overlapPercent ?? -1.0)
|
|
2234
|
+
let state = IncrementalStateObject(
|
|
2235
|
+
panoramaPath: nil,
|
|
2236
|
+
width: 0,
|
|
2237
|
+
height: 0,
|
|
2238
|
+
acceptedCount: acceptedCount,
|
|
2239
|
+
outcome: outcome,
|
|
2240
|
+
confidence: 0,
|
|
2241
|
+
overlapPercent: overlapPercent,
|
|
2242
|
+
processingMs: 0,
|
|
2243
|
+
isLandscape: prev?.isLandscape ?? false,
|
|
2244
|
+
paintedExtent: prev?.paintedExtent ?? 0,
|
|
2245
|
+
panExtent: prev?.panExtent ?? 0,
|
|
2246
|
+
keyframeMax: decision.maxCount
|
|
2247
|
+
)
|
|
2248
|
+
stateLock.lock()
|
|
2249
|
+
self.lastState = state
|
|
2250
|
+
stateLock.unlock()
|
|
2251
|
+
NotificationCenter.default.post(
|
|
2252
|
+
name: .retailensIncrementalStateUpdate,
|
|
2253
|
+
object: nil,
|
|
2254
|
+
userInfo: state.asDictionary()
|
|
2255
|
+
)
|
|
2256
|
+
}
|
|
2257
|
+
|
|
2258
|
+
/// Read the most recent state snapshot (JS pulls this on demand).
|
|
2259
|
+
@objc public func currentStateDictionary() -> [String: Any]? {
|
|
2260
|
+
stateLock.lock()
|
|
2261
|
+
defer { stateLock.unlock() }
|
|
2262
|
+
return lastState?.asDictionary()
|
|
2263
|
+
}
|
|
2264
|
+
|
|
2265
|
+
// ── ARSession frame consumer hook ───────────────────────────────
|
|
2266
|
+
|
|
2267
|
+
/// Called from the ARSession delegate while the engine is active.
|
|
2268
|
+
/// MUST consume the pixel buffer before returning (Apple's pool
|
|
2269
|
+
/// reuse contract — see comments in RNSARSession).
|
|
2270
|
+
@objc public func consumeFrame(
|
|
2271
|
+
pixelBuffer: CVPixelBuffer,
|
|
2272
|
+
pose: RNSARFramePose
|
|
2273
|
+
) {
|
|
2274
|
+
guard stateLock.try() else {
|
|
2275
|
+
// start/stop in flight — drop this frame.
|
|
2276
|
+
return
|
|
2277
|
+
}
|
|
2278
|
+
let hybrid = self.hybridEngine
|
|
2279
|
+
let slit = self.firstwinsEngine
|
|
2280
|
+
let isRunning = self.isRunning
|
|
2281
|
+
// V16 Phase 1 — capture batch-keyframe state under the lock so
|
|
2282
|
+
// the work-queue closure (or the synchronous reject below)
|
|
2283
|
+
// sees consistent ivars even if start/cancel races.
|
|
2284
|
+
let inBatchKeyframeMode = self.batchKeyframeMode
|
|
2285
|
+
let collector = self.keyframeCollector
|
|
2286
|
+
let rotationDegreesForBatch = self.keyframeRotationDegrees
|
|
2287
|
+
let exifOrientationForBatch = self.keyframeExifOrientation
|
|
2288
|
+
|
|
2289
|
+
// V13.0c.1 — diagnostic translation logging. Captures the
|
|
2290
|
+
// FIRST frame's world position, then logs delta from first
|
|
2291
|
+
// on every subsequent frame. Throttled to every 5th call
|
|
2292
|
+
// to keep Console.app readable. This data tells us how
|
|
2293
|
+
// much users physically translate during typical captures
|
|
2294
|
+
// before we commit to per-pixel depth correction (V13.0c.2+).
|
|
2295
|
+
//
|
|
2296
|
+
// Notes:
|
|
2297
|
+
// • tx,ty,tz are in ARKit world coords (metres).
|
|
2298
|
+
// • magnitudeM = √(Δtx² + Δty² + Δtz²) — total camera
|
|
2299
|
+
// displacement from first frame.
|
|
2300
|
+
// • If typical magnitudeM < 0.05 m (5 cm) → minimal
|
|
2301
|
+
// translation, NCC alone may suffice.
|
|
2302
|
+
// • If typical magnitudeM > 0.30 m (30 cm) → significant
|
|
2303
|
+
// translation, per-depth correction essential.
|
|
2304
|
+
if isRunning {
|
|
2305
|
+
self.consumeFrameCounter += 1
|
|
2306
|
+
if !self.hasFirstFrameTranslation {
|
|
2307
|
+
self.firstFrameTx = pose.tx
|
|
2308
|
+
self.firstFrameTy = pose.ty
|
|
2309
|
+
self.firstFrameTz = pose.tz
|
|
2310
|
+
self.hasFirstFrameTranslation = true
|
|
2311
|
+
// V13.0c.1.1 — FAULT-level os_log under same subsystem
|
|
2312
|
+
// as V13.0b-gate so logs appear under either Console.app
|
|
2313
|
+
// filter (process-only or subsystem).
|
|
2314
|
+
os_log(.fault, log: Self.diagLog,
|
|
2315
|
+
"[V13.0c-trans] first-frame world position tx=%.4f ty=%.4f tz=%.4f",
|
|
2316
|
+
pose.tx, pose.ty, pose.tz)
|
|
2317
|
+
} else if self.consumeFrameCounter % 5 == 0 {
|
|
2318
|
+
let dx = pose.tx - self.firstFrameTx
|
|
2319
|
+
let dy = pose.ty - self.firstFrameTy
|
|
2320
|
+
let dz = pose.tz - self.firstFrameTz
|
|
2321
|
+
let mag = sqrt(dx * dx + dy * dy + dz * dz)
|
|
2322
|
+
os_log(.fault, log: Self.diagLog,
|
|
2323
|
+
"[V13.0c-trans] #%d delta_t_world=(%+.4f,%+.4f,%+.4f) magnitude=%.4f m",
|
|
2324
|
+
self.consumeFrameCounter, dx, dy, dz, mag)
|
|
2325
|
+
}
|
|
2326
|
+
}
|
|
2327
|
+
|
|
2328
|
+
// V16 Phase 2 (C1 fix) — evaluate the pose-driven keyframe gate
|
|
2329
|
+
// BEFORE releasing stateLock.
|
|
2330
|
+
//
|
|
2331
|
+
// RCA: KeyframeGate is documented thread-unsafe (its C++ pImpl
|
|
2332
|
+
// holds std::optional<std::vector<Vec2>>, etc.; caller must
|
|
2333
|
+
// serialise). Every OTHER access already runs under stateLock —
|
|
2334
|
+
// `start`'s configure-gate path, `cancel`'s `keyframeGate.reset()`,
|
|
2335
|
+
// `markNextFrameAsLastKeyframe`'s `forceAcceptNext = true` write.
|
|
2336
|
+
// ONLY this read site (the AR-delegate hot path) used to run
|
|
2337
|
+
// OUTSIDE stateLock, which is a thread-safety hole — concurrent
|
|
2338
|
+
// mutation from the JS-bridge thread could corrupt the gate's
|
|
2339
|
+
// internal vectors mid-evaluate().
|
|
2340
|
+
//
|
|
2341
|
+
// We compute the decision under the lock, then unlock BEFORE the
|
|
2342
|
+
// heavier work (emitKeyframeRejectState fires a JS event; the
|
|
2343
|
+
// workQueue dispatch + JPEG encode + path-append happen below).
|
|
2344
|
+
// Holding the lock for the ~50–200 µs evaluate is negligible vs
|
|
2345
|
+
// the alternative (latent corruption that produced the V16
|
|
2346
|
+
// Phase 1b finalize crashes).
|
|
2347
|
+
//
|
|
2348
|
+
// We evaluate BEFORE the workInFlight check so a rejected frame
|
|
2349
|
+
// doesn't burn workQueue slots — the gate is the cheap filter,
|
|
2350
|
+
// the engine is the expensive one.
|
|
2351
|
+
var gateDecision: KeyframeGateDecision? = nil
|
|
2352
|
+
// V16 — eval-throttle. When flowEvalEveryNFrames > 1, run the
|
|
2353
|
+
// gate every Nth consumeFrame instead of every frame. Cuts
|
|
2354
|
+
// CPU on the AR delegate path linearly with N at the cost of
|
|
2355
|
+
// up to N-1 frames of acceptance latency. Doesn't change
|
|
2356
|
+
// WHICH frames are accepted — just when we check.
|
|
2357
|
+
//
|
|
2358
|
+
// `consumeFrameCounter` was already incremented above (line
|
|
2359
|
+
// ~1596) so the first frame (counter=1) is always evaluated
|
|
2360
|
+
// regardless of N: (1-1) % N == 0 for any N ≥ 1. Subsequent
|
|
2361
|
+
// evals land on counter = 1+N, 1+2N, ... — first frame
|
|
2362
|
+
// triggers immediately, then every Nth one after.
|
|
2363
|
+
let evalCadence = max(1, self.keyframeGate.flowEvalEveryNFrames)
|
|
2364
|
+
let cadenceFires = ((self.consumeFrameCounter - 1) % evalCadence == 0)
|
|
2365
|
+
let gateActive =
|
|
2366
|
+
isRunning
|
|
2367
|
+
&& (hybrid != nil || slit != nil || inBatchKeyframeMode)
|
|
2368
|
+
&& self.keyframeGate.enabled
|
|
2369
|
+
let shouldEvaluateGate = gateActive && cadenceFires
|
|
2370
|
+
// True iff the gate is active for this capture but we're
|
|
2371
|
+
// skipping THIS specific frame due to the throttle. In that
|
|
2372
|
+
// case we must also skip the workQueue save path below —
|
|
2373
|
+
// otherwise non-Nth frames would be unconditionally saved as
|
|
2374
|
+
// keyframes, which would defeat the gate entirely.
|
|
2375
|
+
let throttledThisFrame = gateActive && !cadenceFires
|
|
2376
|
+
if shouldEvaluateGate {
|
|
2377
|
+
let plane = RNSARSession.shared.latchedPlaneTransform()
|
|
2378
|
+
// V16 A2 — call the pixel-buffer-aware overload so Flow
|
|
2379
|
+
// strategy gets the image content. Pose strategy is
|
|
2380
|
+
// routed to the fast pose-only path inside the bridge,
|
|
2381
|
+
// so we don't pay the buffer-lock cost for Pose frames.
|
|
2382
|
+
gateDecision = self.keyframeGate.evaluate(
|
|
2383
|
+
pose: pose,
|
|
2384
|
+
latchedPlane: plane,
|
|
2385
|
+
pixelBuffer: pixelBuffer
|
|
2386
|
+
)
|
|
2387
|
+
}
|
|
2388
|
+
stateLock.unlock()
|
|
2389
|
+
|
|
2390
|
+
// V16 eval-throttle bail. If the gate is active but we
|
|
2391
|
+
// skipped evaluation for this frame, drop the entire save
|
|
2392
|
+
// pipeline. We emit no event and don't burn the workQueue
|
|
2393
|
+
// slot — the next AR-delegate frame that lands on the
|
|
2394
|
+
// cadence will go through normally.
|
|
2395
|
+
if throttledThisFrame {
|
|
2396
|
+
return
|
|
2397
|
+
}
|
|
2398
|
+
|
|
2399
|
+
// 2026-05-15 — `[V16-keyframe-decision]` per-decision diag log
|
|
2400
|
+
// removed. Original commit 9dd0ae9 (formerly 7664d5a pre-
|
|
2401
|
+
// author-rewrite) added a fault-level os_log on every gate
|
|
2402
|
+
// decision (accept + reject) to investigate the post-fix-7
|
|
2403
|
+
// bursting. Root cause was identified + fixed in fix-9/10/11
|
|
2404
|
+
// (see 2026-05-12-finalize-crash-investigation.md, Round 3).
|
|
2405
|
+
// The log was noise after that; removing it keeps the
|
|
2406
|
+
// fault-level Console output clean. Reject-path decisions
|
|
2407
|
+
// still emit via emitKeyframeRejectState() below for UI pill
|
|
2408
|
+
// text; accept-path decisions emit via the keyframeAccepted
|
|
2409
|
+
// path and the JS state subscriber.
|
|
2410
|
+
|
|
2411
|
+
// V16 Phase 1 — batch-keyframe is also a valid running mode
|
|
2412
|
+
// (no engine pointer, but the collector and gate are active).
|
|
2413
|
+
guard isRunning,
|
|
2414
|
+
(hybrid != nil || slit != nil || inBatchKeyframeMode)
|
|
2415
|
+
else { return }
|
|
2416
|
+
|
|
2417
|
+
// Surface the gate's reject decision (if any) outside the lock.
|
|
2418
|
+
// emitKeyframeRejectState dispatches a JS bridge event which
|
|
2419
|
+
// could itself acquire other locks; keeping it outside stateLock
|
|
2420
|
+
// is the safe call.
|
|
2421
|
+
if let decision = gateDecision, !decision.accept {
|
|
2422
|
+
self.emitKeyframeRejectState(decision: decision)
|
|
2423
|
+
return
|
|
2424
|
+
}
|
|
2425
|
+
|
|
2426
|
+
// Compute yaw + pitch from the quaternion. Convention:
|
|
2427
|
+
// yaw = rotation about world Y (camera turning left/right)
|
|
2428
|
+
// pitch = rotation about camera X (camera tilting up/down)
|
|
2429
|
+
let q = simd_quatf(
|
|
2430
|
+
ix: Float(pose.qx), iy: Float(pose.qy),
|
|
2431
|
+
iz: Float(pose.qz), r: Float(pose.qw)
|
|
2432
|
+
)
|
|
2433
|
+
let (yaw, pitch) = Self.yawPitch(from: q)
|
|
2434
|
+
|
|
2435
|
+
// Both FoVs from physical camera intrinsics. Passing the
|
|
2436
|
+
// PHYSICAL vertical FoV (vs deriving it from compose aspect
|
|
2437
|
+
// inside the engine) is what fixes the v1/v2 "only left-to-
|
|
2438
|
+
// right portrait pan responds" bug — the engine's overlap
|
|
2439
|
+
// gate compared world-pitch against a compose-aspect-derived
|
|
2440
|
+
// vertical FoV that didn't match the actual camera, so most
|
|
2441
|
+
// top-to-bottom pans fell outside the 30-70% window.
|
|
2442
|
+
let fovHRad = 2.0 * atan(Double(pose.imageWidth) / (2.0 * pose.fx))
|
|
2443
|
+
let fovVRad = 2.0 * atan(Double(pose.imageHeight) / (2.0 * pose.fy))
|
|
2444
|
+
let fovHDeg = fovHRad * 180.0 / .pi
|
|
2445
|
+
let fovVDeg = fovVRad * 180.0 / .pi
|
|
2446
|
+
|
|
2447
|
+
let trackingPoor = (pose.trackingState != .tracking)
|
|
2448
|
+
|
|
2449
|
+
// V11 Gap #27: dispatch the heavy pipeline (engine.ingest +
|
|
2450
|
+
// optional snapshot) to the work queue. Earlier versions
|
|
2451
|
+
// ran the full ~70 ms accept inside the AR delegate thread,
|
|
2452
|
+
// blocking ARKit's 16 ms inter-frame budget and causing
|
|
2453
|
+
// ~4-5 frames to be dropped during each accept.
|
|
2454
|
+
//
|
|
2455
|
+
// Backpressure: if the work queue is already busy with a
|
|
2456
|
+
// previous frame, drop this one (don't queue up — that'd
|
|
2457
|
+
// produce an ever-growing latency between AR-time and
|
|
2458
|
+
// canvas-state). CVPixelBuffer auto-retains via Swift's
|
|
2459
|
+
// ARC; the closure capture extends its lifetime past the
|
|
2460
|
+
// delegate return.
|
|
2461
|
+
if self.workInFlight {
|
|
2462
|
+
self.droppedBackpressure += 1
|
|
2463
|
+
return
|
|
2464
|
+
}
|
|
2465
|
+
self.workInFlight = true
|
|
2466
|
+
|
|
2467
|
+
// V16 Phase 1b.fix2 — DEEP COPY of the pixel buffer.
|
|
2468
|
+
//
|
|
2469
|
+
// Apple's contract on ARFrame.capturedImage: "valid only
|
|
2470
|
+
// within the scope of the captured ARFrame. To use beyond
|
|
2471
|
+
// that scope you must make a copy." Swift's `let pbCopy =
|
|
2472
|
+
// pixelBuffer` is JUST an ARC retain on the CVPixelBufferRef;
|
|
2473
|
+
// it does NOT extend the lifetime of the underlying IOSurface.
|
|
2474
|
+
// ARKit's pixel-buffer pool (~3–4 buffers) recycles slots
|
|
2475
|
+
// aggressively under load — long pans race the workQueue's
|
|
2476
|
+
// 50–100 ms JPEG encode against pool churn and randomly hit
|
|
2477
|
+
// a freed slot, producing the EXC_BAD_ACCESS in objc_retain
|
|
2478
|
+
// we saw mid-pan ("when I pan the device more").
|
|
2479
|
+
//
|
|
2480
|
+
// CVPixelBufferCreate + memcpy gives us a fully-owned copy
|
|
2481
|
+
// that ARC alone governs. ~1-2 ms cost on iPhone 16 Pro
|
|
2482
|
+
// (10 MB memcpy at memory bandwidth ~10 GB/s). Fixes the
|
|
2483
|
+
// crash for ALL engine paths (slit-scan / hybrid / batch-
|
|
2484
|
+
// keyframe) since they all dispatch via consumeFrame.
|
|
2485
|
+
guard let pbCopy = Self.deepCopyPixelBuffer(pixelBuffer) else {
|
|
2486
|
+
// Allocation failure — drop the frame. Extremely rare;
|
|
2487
|
+
// would only happen under genuine OOM.
|
|
2488
|
+
os_log(.fault, log: Self.diagLog,
|
|
2489
|
+
"[V16-pbcopy] CVPixelBufferCreate failed; dropping frame")
|
|
2490
|
+
self.workInFlight = false
|
|
2491
|
+
return
|
|
2492
|
+
}
|
|
2493
|
+
workQueue.async { [weak self] in
|
|
2494
|
+
defer { self?.workInFlight = false }
|
|
2495
|
+
guard let self = self else { return }
|
|
2496
|
+
|
|
2497
|
+
// V12.1 frame-leak fix: finalize/cancel may have run on
|
|
2498
|
+
// the JS thread between consumeFrame's isRunning check
|
|
2499
|
+
// and now. If isRunning is now false, this frame would
|
|
2500
|
+
// have been dispatched a few ms BEFORE the user released
|
|
2501
|
+
// the shutter — ingesting it now is exactly the
|
|
2502
|
+
// "phantom frame after release" the user observed. Bail
|
|
2503
|
+
// before touching the engine.
|
|
2504
|
+
self.stateLock.lock()
|
|
2505
|
+
let stillRunning = self.isRunning
|
|
2506
|
+
self.stateLock.unlock()
|
|
2507
|
+
guard stillRunning else { return }
|
|
2508
|
+
|
|
2509
|
+
// V16 Phase 1 — batch-keyframe path: save the buffer as
|
|
2510
|
+
// a JPEG via the collector, append the pose, emit a
|
|
2511
|
+
// notification so JS can render the thumbnail in
|
|
2512
|
+
// LiveFrameStrip. No incremental engine to call.
|
|
2513
|
+
if inBatchKeyframeMode {
|
|
2514
|
+
guard let coll = collector else { return }
|
|
2515
|
+
do {
|
|
2516
|
+
let record = try coll.saveKeyframe(
|
|
2517
|
+
pbCopy,
|
|
2518
|
+
rotationDegrees: rotationDegreesForBatch,
|
|
2519
|
+
exifOrientation: exifOrientationForBatch,
|
|
2520
|
+
jpegQuality: 80
|
|
2521
|
+
)
|
|
2522
|
+
self.stateLock.lock()
|
|
2523
|
+
self.keyframePaths.append(record.path)
|
|
2524
|
+
self.keyframePoses.append(pose.asDictionary())
|
|
2525
|
+
let count = self.keyframePaths.count
|
|
2526
|
+
self.stateLock.unlock()
|
|
2527
|
+
os_log(.fault, log: Self.diagLog,
|
|
2528
|
+
"[V16-batch-keyframe] saved keyframe %d → %{public}@ (%dx%d)",
|
|
2529
|
+
Int32(count),
|
|
2530
|
+
record.path,
|
|
2531
|
+
Int32(record.width), Int32(record.height))
|
|
2532
|
+
self.emitBatchKeyframeAcceptedState(
|
|
2533
|
+
thumbnailPath: record.path,
|
|
2534
|
+
keyframeIndex: Int(record.index),
|
|
2535
|
+
keyframeCount: count,
|
|
2536
|
+
keyframeMax: self.keyframeGate.maxCount,
|
|
2537
|
+
isLandscape: pose.imageWidth >= pose.imageHeight
|
|
2538
|
+
)
|
|
2539
|
+
} catch let err as NSError {
|
|
2540
|
+
os_log(.fault, log: Self.diagLog,
|
|
2541
|
+
"[V16-batch-keyframe] saveKeyframe failed: %{public}@",
|
|
2542
|
+
err.localizedDescription)
|
|
2543
|
+
}
|
|
2544
|
+
return
|
|
2545
|
+
}
|
|
2546
|
+
|
|
2547
|
+
// V15.0b — if a vertical plane has just been detected and
|
|
2548
|
+
// we haven't propagated it to the slit-scan engine yet,
|
|
2549
|
+
// do so now. Propagated only once per latched plane;
|
|
2550
|
+
// RNSARSession resets on stop().
|
|
2551
|
+
if !self.havePropagatedPlane,
|
|
2552
|
+
let plane = RNSARSession.shared.planeTransformFlat() {
|
|
2553
|
+
slit?.setPlaneTransformFlat(plane)
|
|
2554
|
+
self.havePropagatedPlane = true
|
|
2555
|
+
// V15.0c.4 — fault log so we can see the propagation
|
|
2556
|
+
// moment without rate-limit drops.
|
|
2557
|
+
os_log(.fault, log: Self.diagLog,
|
|
2558
|
+
"[V15.0b-plane] bridge propagated plane to slit-scan engine (one-shot per capture)")
|
|
2559
|
+
}
|
|
2560
|
+
|
|
2561
|
+
let telemetry: RLISFrameTelemetry
|
|
2562
|
+
if let hybrid = hybrid {
|
|
2563
|
+
telemetry = hybrid.ingest(
|
|
2564
|
+
pixelBuffer: pbCopy, qx: pose.qx, qy: pose.qy, qz: pose.qz, qw: pose.qw,
|
|
2565
|
+
tx: pose.tx, ty: pose.ty, tz: pose.tz,
|
|
2566
|
+
fx: pose.fx, fy: pose.fy, cx: pose.cx, cy: pose.cy,
|
|
2567
|
+
imageWidth: pose.imageWidth, imageHeight: pose.imageHeight,
|
|
2568
|
+
yaw: yaw, pitch: pitch,
|
|
2569
|
+
fovHorizDegrees: fovHDeg, fovVertDegrees: fovVDeg,
|
|
2570
|
+
trackingPoor: trackingPoor
|
|
2571
|
+
)
|
|
2572
|
+
} else if let slit = slit {
|
|
2573
|
+
// V13.0e — slit-scan engine consumes tx/ty/tz for
|
|
2574
|
+
// ORB-triangulation-based depth estimation and per-frame
|
|
2575
|
+
// translation parallax correction. Hybrid passes them
|
|
2576
|
+
// for API symmetry; only the slit engine uses them.
|
|
2577
|
+
telemetry = slit.ingest(
|
|
2578
|
+
pixelBuffer: pbCopy, qx: pose.qx, qy: pose.qy, qz: pose.qz, qw: pose.qw,
|
|
2579
|
+
tx: pose.tx, ty: pose.ty, tz: pose.tz,
|
|
2580
|
+
fx: pose.fx, fy: pose.fy, cx: pose.cx, cy: pose.cy,
|
|
2581
|
+
imageWidth: pose.imageWidth, imageHeight: pose.imageHeight,
|
|
2582
|
+
yaw: yaw, pitch: pitch,
|
|
2583
|
+
fovHorizDegrees: fovHDeg, fovVertDegrees: fovVDeg,
|
|
2584
|
+
trackingPoor: trackingPoor
|
|
2585
|
+
)
|
|
2586
|
+
} else {
|
|
2587
|
+
return
|
|
2588
|
+
}
|
|
2589
|
+
|
|
2590
|
+
self.processIngestResult(
|
|
2591
|
+
telemetry: telemetry, hybrid: hybrid, slit: slit)
|
|
2592
|
+
}
|
|
2593
|
+
}
|
|
2594
|
+
|
|
2595
|
+
/// Pulled out of consumeFrame so the work-queue closure stays
|
|
2596
|
+
/// readable. Same flow: build state, optionally snapshot, post
|
|
2597
|
+
/// notification.
|
|
2598
|
+
private func processIngestResult(
|
|
2599
|
+
telemetry: RLISFrameTelemetry,
|
|
2600
|
+
hybrid: OpenCVIncrementalStitcher?,
|
|
2601
|
+
slit: OpenCVFirstWinsCylindricalStitcher?
|
|
2602
|
+
) {
|
|
2603
|
+
var snapshotPath: String?
|
|
2604
|
+
var snapW = 0, snapH = 0
|
|
2605
|
+
let outcome = IncrementalOutcome(rawValue: telemetry.outcome.rawValue)
|
|
2606
|
+
?? .skippedTrackingPoor
|
|
2607
|
+
|
|
2608
|
+
let isAccept = (telemetry.outcome == .acceptedHigh ||
|
|
2609
|
+
telemetry.outcome == .acceptedMedium)
|
|
2610
|
+
|
|
2611
|
+
if isAccept {
|
|
2612
|
+
self.acceptsSinceSnapshot += 1
|
|
2613
|
+
if self.acceptsSinceSnapshot >= self.snapshotEveryNAccepts {
|
|
2614
|
+
self.acceptsSinceSnapshot = 0
|
|
2615
|
+
do {
|
|
2616
|
+
let snap: RLISSnapshot
|
|
2617
|
+
if let hybrid = hybrid {
|
|
2618
|
+
snap = try hybrid.snapshot(
|
|
2619
|
+
withJpegQuality: self.snapshotJpegQuality)
|
|
2620
|
+
} else {
|
|
2621
|
+
snap = try slit!.snapshot(
|
|
2622
|
+
withJpegQuality: self.snapshotJpegQuality)
|
|
2623
|
+
}
|
|
2624
|
+
snapshotPath = snap.panoramaPath
|
|
2625
|
+
snapW = snap.width
|
|
2626
|
+
snapH = snap.height
|
|
2627
|
+
} catch {
|
|
2628
|
+
// Silently dropping a snapshot is fine — next
|
|
2629
|
+
// accept will retry.
|
|
2630
|
+
}
|
|
2631
|
+
}
|
|
2632
|
+
}
|
|
2633
|
+
|
|
2634
|
+
// V16 — pass the gate's max keyframe count when the gate is
|
|
2635
|
+
// active so JS can render "Keyframes: n/max". Zero signals
|
|
2636
|
+
// "gate disabled" to the JS pill.
|
|
2637
|
+
let kfMax = self.keyframeGate.enabled ? self.keyframeGate.maxCount : 0
|
|
2638
|
+
let state = IncrementalStateObject(
|
|
2639
|
+
panoramaPath: snapshotPath,
|
|
2640
|
+
width: snapW,
|
|
2641
|
+
height: snapH,
|
|
2642
|
+
acceptedCount: hybrid?.acceptedCount ?? slit?.acceptedCount ?? 0,
|
|
2643
|
+
outcome: outcome,
|
|
2644
|
+
confidence: telemetry.confidence,
|
|
2645
|
+
overlapPercent: telemetry.overlapPercent,
|
|
2646
|
+
processingMs: telemetry.processingMs,
|
|
2647
|
+
isLandscape: telemetry.isLandscape,
|
|
2648
|
+
paintedExtent: telemetry.paintedExtent,
|
|
2649
|
+
panExtent: telemetry.panExtent,
|
|
2650
|
+
keyframeMax: kfMax
|
|
2651
|
+
)
|
|
2652
|
+
stateLock.lock()
|
|
2653
|
+
self.lastState = state
|
|
2654
|
+
stateLock.unlock()
|
|
2655
|
+
|
|
2656
|
+
// Emit always — JS may want to drive UX on rejects too.
|
|
2657
|
+
// NotificationCenter is thread-agnostic; the bridge converts
|
|
2658
|
+
// it to a main-thread RN event.
|
|
2659
|
+
NotificationCenter.default.post(
|
|
2660
|
+
name: .retailensIncrementalStateUpdate,
|
|
2661
|
+
object: nil,
|
|
2662
|
+
userInfo: state.asDictionary()
|
|
2663
|
+
)
|
|
2664
|
+
}
|
|
2665
|
+
|
|
2666
|
+
// ── Debug log file ──────────────────────────────────────────────
|
|
2667
|
+
//
|
|
2668
|
+
// iOS Console rate-limits Swift NSLog at 60 Hz, dropping most
|
|
2669
|
+
// output silently. File-based log captures everything, and we
|
|
2670
|
+
// can pull it from Xcode's device-container browser. Path:
|
|
2671
|
+
// <Documents>/rlis-debug.log
|
|
2672
|
+
private static let debugLogQueue = DispatchQueue(label: "rlis.debuglog")
|
|
2673
|
+
private static var debugLogPath: String = {
|
|
2674
|
+
let docs = NSSearchPathForDirectoriesInDomains(
|
|
2675
|
+
.documentDirectory, .userDomainMask, true).first ?? NSTemporaryDirectory()
|
|
2676
|
+
return (docs as NSString).appendingPathComponent("rlis-debug.log")
|
|
2677
|
+
}()
|
|
2678
|
+
static func fileLog(_ msg: String) {
|
|
2679
|
+
let line = "\(Date().timeIntervalSince1970): \(msg)\n"
|
|
2680
|
+
debugLogQueue.async {
|
|
2681
|
+
let data = line.data(using: .utf8) ?? Data()
|
|
2682
|
+
if let fh = FileHandle(forWritingAtPath: debugLogPath) {
|
|
2683
|
+
fh.seekToEndOfFile()
|
|
2684
|
+
fh.write(data)
|
|
2685
|
+
fh.closeFile()
|
|
2686
|
+
} else {
|
|
2687
|
+
try? data.write(to: URL(fileURLWithPath: debugLogPath),
|
|
2688
|
+
options: .atomicWrite)
|
|
2689
|
+
}
|
|
2690
|
+
}
|
|
2691
|
+
// Also try NSLog in case it does work occasionally
|
|
2692
|
+
NSLog("[RLIS-PIP] %@", msg)
|
|
2693
|
+
}
|
|
2694
|
+
|
|
2695
|
+
// ── Helpers ─────────────────────────────────────────────────────
|
|
2696
|
+
|
|
2697
|
+
/// Extract yaw (rotation about world Y) and pitch (rotation about
|
|
2698
|
+
/// camera X) from an ARKit camera quaternion. Numerically stable
|
|
2699
|
+
/// for camera orientations the user holds in practice — straight
|
|
2700
|
+
/// up/down is gimbal-locked but a shelf-audit user is never there.
|
|
2701
|
+
private static func yawPitch(from q: simd_quatf) -> (Double, Double) {
|
|
2702
|
+
// Apply the quaternion to ARKit's camera-forward vector
|
|
2703
|
+
// (-Z in camera frame) to get the camera-forward in world.
|
|
2704
|
+
// Yaw is the angle of the projection onto the X-Z plane;
|
|
2705
|
+
// pitch is the elevation angle.
|
|
2706
|
+
let forward = simd_act(q, simd_float3(0, 0, -1))
|
|
2707
|
+
let yaw = Double(atan2(forward.x, -forward.z))
|
|
2708
|
+
let pitch = Double(asin(forward.y))
|
|
2709
|
+
return (yaw, pitch)
|
|
2710
|
+
}
|
|
2711
|
+
}
|
|
2712
|
+
|
|
2713
|
+
// ── Bridge contract for ARSession ───────────────────────────────────
|
|
2714
|
+
//
|
|
2715
|
+
// The AR session calls into us via an @objc protocol so the dependency
|
|
2716
|
+
// arrow points the right way: ARSession (low-level) delivers frames
|
|
2717
|
+
// to a consumer it knows nothing about. The Stitcher implements the
|
|
2718
|
+
// protocol and registers itself.
|
|
2719
|
+
|
|
2720
|
+
@objc public protocol ARFrameConsumer: AnyObject {
|
|
2721
|
+
/// Called on the ARSession delegate's queue. The pixel buffer is
|
|
2722
|
+
/// only valid for the duration of this call (Apple's ARKit pool
|
|
2723
|
+
/// reuse contract); consumers must copy out before returning.
|
|
2724
|
+
func consumeFrame(pixelBuffer: CVPixelBuffer, pose: RNSARFramePose)
|
|
2725
|
+
}
|
|
2726
|
+
|
|
2727
|
+
extension IncrementalStitcher: ARFrameConsumer {}
|