react-native-image-stitcher 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (151) hide show
  1. package/CHANGELOG.md +96 -0
  2. package/LICENSE +201 -0
  3. package/NOTICE +21 -0
  4. package/README.md +189 -0
  5. package/RNImageStitcher.podspec +76 -0
  6. package/android/build.gradle +224 -0
  7. package/android/src/main/AndroidManifest.xml +3 -0
  8. package/android/src/main/cpp/CMakeLists.txt +124 -0
  9. package/android/src/main/cpp/image_stitcher_jni.cpp +145 -0
  10. package/android/src/main/cpp/keyframe_gate_jni.cpp +204 -0
  11. package/android/src/main/java/io/imagestitcher/rn/BatchStitcher.kt +426 -0
  12. package/android/src/main/java/io/imagestitcher/rn/IncrementalFirstwinsEngine.kt +960 -0
  13. package/android/src/main/java/io/imagestitcher/rn/IncrementalStitcher.kt +2371 -0
  14. package/android/src/main/java/io/imagestitcher/rn/KeyframeGate.kt +256 -0
  15. package/android/src/main/java/io/imagestitcher/rn/QualityChecker.kt +167 -0
  16. package/android/src/main/java/io/imagestitcher/rn/RNImageStitcherPackage.kt +39 -0
  17. package/android/src/main/java/io/imagestitcher/rn/RNSARCameraView.kt +558 -0
  18. package/android/src/main/java/io/imagestitcher/rn/RNSARCameraViewManager.kt +35 -0
  19. package/android/src/main/java/io/imagestitcher/rn/RNSARSession.kt +784 -0
  20. package/android/src/main/java/io/imagestitcher/rn/ar/BackgroundRenderer.kt +176 -0
  21. package/android/src/main/java/io/imagestitcher/rn/ar/ShaderUtil.kt +67 -0
  22. package/android/src/main/java/io/imagestitcher/rn/ar/YuvImageConverter.kt +201 -0
  23. package/cpp/ar_frame_pose.h +63 -0
  24. package/cpp/keyframe_gate.cpp +927 -0
  25. package/cpp/keyframe_gate.hpp +240 -0
  26. package/cpp/stitcher.cpp +2207 -0
  27. package/cpp/stitcher.hpp +275 -0
  28. package/dist/ar/useARSession.d.ts +102 -0
  29. package/dist/ar/useARSession.js +133 -0
  30. package/dist/camera/ARCameraView.d.ts +93 -0
  31. package/dist/camera/ARCameraView.js +170 -0
  32. package/dist/camera/Camera.d.ts +134 -0
  33. package/dist/camera/Camera.js +688 -0
  34. package/dist/camera/CameraShutter.d.ts +80 -0
  35. package/dist/camera/CameraShutter.js +237 -0
  36. package/dist/camera/CameraView.d.ts +65 -0
  37. package/dist/camera/CameraView.js +117 -0
  38. package/dist/camera/CaptureControlsBar.d.ts +87 -0
  39. package/dist/camera/CaptureControlsBar.js +82 -0
  40. package/dist/camera/CaptureHeader.d.ts +62 -0
  41. package/dist/camera/CaptureHeader.js +81 -0
  42. package/dist/camera/CapturePreview.d.ts +70 -0
  43. package/dist/camera/CapturePreview.js +188 -0
  44. package/dist/camera/CaptureStatusOverlay.d.ts +75 -0
  45. package/dist/camera/CaptureStatusOverlay.js +326 -0
  46. package/dist/camera/CaptureThumbnailStrip.d.ts +87 -0
  47. package/dist/camera/CaptureThumbnailStrip.js +177 -0
  48. package/dist/camera/IncrementalPanGuide.d.ts +83 -0
  49. package/dist/camera/IncrementalPanGuide.js +267 -0
  50. package/dist/camera/PanoramaBandOverlay.d.ts +107 -0
  51. package/dist/camera/PanoramaBandOverlay.js +399 -0
  52. package/dist/camera/PanoramaConfirmModal.d.ts +57 -0
  53. package/dist/camera/PanoramaConfirmModal.js +128 -0
  54. package/dist/camera/PanoramaGuidance.d.ts +79 -0
  55. package/dist/camera/PanoramaGuidance.js +246 -0
  56. package/dist/camera/PanoramaSettingsModal.d.ts +311 -0
  57. package/dist/camera/PanoramaSettingsModal.js +611 -0
  58. package/dist/camera/ViewportCropOverlay.d.ts +46 -0
  59. package/dist/camera/ViewportCropOverlay.js +67 -0
  60. package/dist/camera/useCapture.d.ts +111 -0
  61. package/dist/camera/useCapture.js +160 -0
  62. package/dist/camera/useDeviceOrientation.d.ts +48 -0
  63. package/dist/camera/useDeviceOrientation.js +131 -0
  64. package/dist/camera/useVideoCapture.d.ts +79 -0
  65. package/dist/camera/useVideoCapture.js +151 -0
  66. package/dist/index.d.ts +26 -0
  67. package/dist/index.js +39 -0
  68. package/dist/quality/normaliseOrientation.d.ts +36 -0
  69. package/dist/quality/normaliseOrientation.js +62 -0
  70. package/dist/quality/runQualityCheck.d.ts +41 -0
  71. package/dist/quality/runQualityCheck.js +98 -0
  72. package/dist/sensors/useIMUTranslationGate.d.ts +70 -0
  73. package/dist/sensors/useIMUTranslationGate.js +235 -0
  74. package/dist/stitching/IncrementalStitcherView.d.ts +41 -0
  75. package/dist/stitching/IncrementalStitcherView.js +157 -0
  76. package/dist/stitching/incremental.d.ts +930 -0
  77. package/dist/stitching/incremental.js +133 -0
  78. package/dist/stitching/stitchFrames.d.ts +55 -0
  79. package/dist/stitching/stitchFrames.js +56 -0
  80. package/dist/stitching/stitchVideo.d.ts +119 -0
  81. package/dist/stitching/stitchVideo.js +57 -0
  82. package/dist/stitching/useIncrementalJSDriver.d.ts +74 -0
  83. package/dist/stitching/useIncrementalJSDriver.js +199 -0
  84. package/dist/stitching/useIncrementalStitcher.d.ts +58 -0
  85. package/dist/stitching/useIncrementalStitcher.js +172 -0
  86. package/dist/types.d.ts +58 -0
  87. package/dist/types.js +15 -0
  88. package/ios/Package.swift +72 -0
  89. package/ios/Sources/RNImageStitcher/ARCameraViewManager.m +33 -0
  90. package/ios/Sources/RNImageStitcher/ARCameraViewManager.swift +40 -0
  91. package/ios/Sources/RNImageStitcher/ARSessionBridge.m +55 -0
  92. package/ios/Sources/RNImageStitcher/ARSessionBridge.swift +149 -0
  93. package/ios/Sources/RNImageStitcher/IncrementalStitcher.swift +2727 -0
  94. package/ios/Sources/RNImageStitcher/IncrementalStitcherBridge.m +85 -0
  95. package/ios/Sources/RNImageStitcher/IncrementalStitcherBridge.swift +625 -0
  96. package/ios/Sources/RNImageStitcher/KeyframeGate.swift +328 -0
  97. package/ios/Sources/RNImageStitcher/KeyframeGateBridge.h +141 -0
  98. package/ios/Sources/RNImageStitcher/KeyframeGateBridge.mm +278 -0
  99. package/ios/Sources/RNImageStitcher/OpenCVIncrementalStitcher.h +473 -0
  100. package/ios/Sources/RNImageStitcher/OpenCVIncrementalStitcher.mm +1326 -0
  101. package/ios/Sources/RNImageStitcher/OpenCVKeyframeCollector.h +97 -0
  102. package/ios/Sources/RNImageStitcher/OpenCVKeyframeCollector.mm +296 -0
  103. package/ios/Sources/RNImageStitcher/OpenCVSlitScanStitcher.h +103 -0
  104. package/ios/Sources/RNImageStitcher/OpenCVSlitScanStitcher.mm +3285 -0
  105. package/ios/Sources/RNImageStitcher/OpenCVStitcher.h +238 -0
  106. package/ios/Sources/RNImageStitcher/OpenCVStitcher.mm +1880 -0
  107. package/ios/Sources/RNImageStitcher/QualityChecker.swift +252 -0
  108. package/ios/Sources/RNImageStitcher/QualityCheckerBridge.m +26 -0
  109. package/ios/Sources/RNImageStitcher/QualityCheckerBridge.swift +72 -0
  110. package/ios/Sources/RNImageStitcher/RNSARCameraView.swift +114 -0
  111. package/ios/Sources/RNImageStitcher/RNSARSession.swift +1111 -0
  112. package/ios/Sources/RNImageStitcher/Stitcher.swift +243 -0
  113. package/ios/Sources/RNImageStitcher/StitcherBridge.m +28 -0
  114. package/ios/Sources/RNImageStitcher/StitcherBridge.swift +246 -0
  115. package/package.json +73 -0
  116. package/react-native.config.js +34 -0
  117. package/scripts/opencv-version.txt +1 -0
  118. package/scripts/postinstall-fetch-binaries.js +286 -0
  119. package/src/ar/useARSession.ts +210 -0
  120. package/src/camera/.gitkeep +0 -0
  121. package/src/camera/ARCameraView.tsx +256 -0
  122. package/src/camera/Camera.tsx +1053 -0
  123. package/src/camera/CameraShutter.tsx +292 -0
  124. package/src/camera/CameraView.tsx +157 -0
  125. package/src/camera/CaptureControlsBar.tsx +204 -0
  126. package/src/camera/CaptureHeader.tsx +184 -0
  127. package/src/camera/CapturePreview.tsx +318 -0
  128. package/src/camera/CaptureStatusOverlay.tsx +391 -0
  129. package/src/camera/CaptureThumbnailStrip.tsx +277 -0
  130. package/src/camera/IncrementalPanGuide.tsx +328 -0
  131. package/src/camera/PanoramaBandOverlay.tsx +498 -0
  132. package/src/camera/PanoramaConfirmModal.tsx +206 -0
  133. package/src/camera/PanoramaGuidance.tsx +327 -0
  134. package/src/camera/PanoramaSettingsModal.tsx +1357 -0
  135. package/src/camera/ViewportCropOverlay.tsx +81 -0
  136. package/src/camera/useCapture.ts +279 -0
  137. package/src/camera/useDeviceOrientation.ts +140 -0
  138. package/src/camera/useVideoCapture.ts +236 -0
  139. package/src/index.ts +53 -0
  140. package/src/quality/.gitkeep +0 -0
  141. package/src/quality/normaliseOrientation.ts +79 -0
  142. package/src/quality/runQualityCheck.ts +131 -0
  143. package/src/sensors/useIMUTranslationGate.ts +347 -0
  144. package/src/stitching/.gitkeep +0 -0
  145. package/src/stitching/IncrementalStitcherView.tsx +198 -0
  146. package/src/stitching/incremental.ts +1021 -0
  147. package/src/stitching/stitchFrames.ts +88 -0
  148. package/src/stitching/stitchVideo.ts +153 -0
  149. package/src/stitching/useIncrementalJSDriver.ts +273 -0
  150. package/src/stitching/useIncrementalStitcher.ts +252 -0
  151. package/src/types.ts +78 -0
@@ -0,0 +1,2727 @@
1
+ // SPDX-License-Identifier: Apache-2.0
2
+ //
3
+ // IncrementalStitcher — Swift-side engine for the live
4
+ // panorama-stitching pipeline introduced in
5
+ // docs/site-content/design/2026-04-30-realtime-incremental-stitching.md.
6
+ //
7
+ // What this file does:
8
+ // - Owns a single `OpenCVIncrementalStitcher` instance
9
+ // - Subscribes to `RNSARSession`'s per-frame ARFrame delivery
10
+ // - Converts ARKit pose → yaw/pitch + horizontal FoV
11
+ // - Dispatches addPixelBuffer onto a serial queue
12
+ // - Posts state updates as Notifications so the RN bridge can fan
13
+ // them out to JS as device events
14
+ //
15
+ // What this file deliberately does NOT do:
16
+ // - Touch OpenCV / cv::* — that's confined to the .mm impl behind
17
+ // the ObjC interface.
18
+ // - Manage UIKit views — the live preview is rendered via a separate
19
+ // ViewManager that watches the snapshot file and re-loads it.
20
+ // - Emit RN events directly — that's the bridge's job. This class
21
+ // stays free of any React.framework dependency so it can be
22
+ // swift-tested in isolation.
23
+ //
24
+ // Threading:
25
+ // ARSession delegates fire on an Apple-owned queue (~60 Hz). Inside
26
+ // the delegate we synchronously convert NV12 → BGR Mat (~5 ms) and
27
+ // compute pose-delta gating; CHEAP rejects (overlap < min, > max)
28
+ // short-circuit before any feature work runs. Heavy candidate
29
+ // processing (ORB + match + RANSAC + warp + blend) hops onto a
30
+ // dedicated serial queue so the AR delegate isn't blocked.
31
+ //
32
+ // Pixel-buffer lifetime:
33
+ // Apple guarantees ARFrame.capturedImage stays valid only within
34
+ // the delegate callback (see the comment on RNSARSession's
35
+ // recording-append path). We therefore consume the buffer
36
+ // inside the delegate (the .mm copies pixels into a cv::Mat — the
37
+ // Mat owns its own heap memory) before returning, even when the
38
+ // actual heavy work is dispatched to the serial queue.
39
+
40
+ import Foundation
41
+ import ARKit
42
+ import simd
43
+ import UIKit
44
+ import os.log
45
+
46
+ /// Public outcome enum mirroring the ObjC `RLISFrameOutcome` so JS
47
+ /// callers can inspect what happened to each frame without crossing
48
+ /// the ObjC++ boundary themselves.
49
+ ///
50
+ /// Values 7+ are emitted from the Swift gate layer (KeyframeGate),
51
+ /// not from the native engine. Keep numeric values in lockstep with
52
+ /// `IncrementalOutcome` in incremental.ts.
53
+ @objc public enum IncrementalOutcome: Int {
54
+ case acceptedHigh = 0
55
+ case acceptedMedium = 1
56
+ case skippedTooClose = 2
57
+ case rejectedTooFar = 3
58
+ case rejectedSceneUniform = 4
59
+ case rejectedAlignmentLost = 5
60
+ case skippedTrackingPoor = 6
61
+ /// V12.11 — engine refused a frame because pan reversed past the
62
+ /// running max along the pan axis. Mirrors the JS-side enum value
63
+ /// that's been there since V12.11. Was missing on the Swift side
64
+ /// previously; the bridge fell back to `.skippedTrackingPoor` for
65
+ /// any rawValue >= 7.
66
+ case rejectedReverseDirection = 7
67
+ /// V16 — KeyframeGate rejected the frame: not enough new content
68
+ /// vs the last accepted keyframe (overlap > 1 - overlapThreshold).
69
+ case skippedKeyframeOverlap = 8
70
+ /// V16 — KeyframeGate rejected the frame: hit `keyframeMaxCount`
71
+ /// for the capture. Host should auto-finalize.
72
+ case skippedKeyframeMaxReached = 9
73
+ }
74
+
75
+ /// State snapshot the bridge re-emits to JS on every accepted frame
76
+ /// (and on rejects when there's a hint to surface).
77
+ @objc(IncrementalStateObject)
78
+ public final class IncrementalStateObject: NSObject {
79
+ @objc public let panoramaPath: String?
80
+ @objc public let width: Int
81
+ @objc public let height: Int
82
+ @objc public let acceptedCount: Int
83
+ @objc public let outcome: IncrementalOutcome
84
+ @objc public let confidence: Double
85
+ @objc public let overlapPercent: Double
86
+ @objc public let processingMs: Double
87
+ /// V12.12 — engine-detected physical orientation, plumbed up
88
+ /// from `RLISFrameTelemetry.isLandscape`. See incremental.ts
89
+ /// for the full rationale (single source of truth across SDK
90
+ /// + host).
91
+ @objc public let isLandscape: Bool
92
+ /// V12.14.9 — running painted extent along the pan axis, in
93
+ /// canvas pixels. Combined with `panExtent`, lets the JS band
94
+ /// overlay size the thumbnail proportionally on every state
95
+ /// event (not just snapshot frames).
96
+ @objc public let paintedExtent: Int
97
+ /// V12.14.9 — total canvas pan-axis extent (engine config).
98
+ /// Constant for the lifetime of a capture. fillRatio =
99
+ /// `paintedExtent / panExtent`.
100
+ @objc public let panExtent: Int
101
+ /// V16 — KeyframeGate's max-keyframes cap for this capture. 0
102
+ /// when the gate is disabled (frameSelectionMode = "time-based"),
103
+ /// in which case `acceptedCount` should be displayed as a raw
104
+ /// counter rather than a "n / max" pill. When > 0, the JS
105
+ /// status pill renders "Keyframes: acceptedCount / keyframeMax".
106
+ @objc public let keyframeMax: Int
107
+
108
+ @objc public init(
109
+ panoramaPath: String?,
110
+ width: Int,
111
+ height: Int,
112
+ acceptedCount: Int,
113
+ outcome: IncrementalOutcome,
114
+ confidence: Double,
115
+ overlapPercent: Double,
116
+ processingMs: Double,
117
+ isLandscape: Bool,
118
+ paintedExtent: Int,
119
+ panExtent: Int,
120
+ keyframeMax: Int = 0
121
+ ) {
122
+ self.panoramaPath = panoramaPath
123
+ self.width = width
124
+ self.height = height
125
+ self.acceptedCount = acceptedCount
126
+ self.outcome = outcome
127
+ self.confidence = confidence
128
+ self.overlapPercent = overlapPercent
129
+ self.processingMs = processingMs
130
+ self.isLandscape = isLandscape
131
+ self.paintedExtent = paintedExtent
132
+ self.panExtent = panExtent
133
+ self.keyframeMax = keyframeMax
134
+ }
135
+
136
+ @objc public func asDictionary() -> [String: Any] {
137
+ var dict: [String: Any] = [
138
+ "width": width,
139
+ "height": height,
140
+ "acceptedCount": acceptedCount,
141
+ "outcome": outcome.rawValue,
142
+ "confidence": confidence,
143
+ "overlapPercent": overlapPercent,
144
+ "processingMs": processingMs,
145
+ "isLandscape": isLandscape,
146
+ "paintedExtent": paintedExtent,
147
+ "panExtent": panExtent,
148
+ "keyframeMax": keyframeMax,
149
+ ]
150
+ if let p = panoramaPath { dict["panoramaPath"] = p }
151
+ return dict
152
+ }
153
+ }
154
+
155
+ /// Notification names — bridge subscribes to these and re-emits as
156
+ /// React Native device events. Keeping the engine framework-free
157
+ /// keeps Swift unit tests viable.
158
+ public extension Notification.Name {
159
+ static let retailensIncrementalStateUpdate =
160
+ Notification.Name("IncrementalStateUpdate")
161
+ }
162
+
163
+
164
+ // MARK: - FinalizePayload (C2 — stateless finalize, fix-attempt 8)
165
+ //
166
+ // Value-typed snapshot of every input the finalize closures need.
167
+ // Built in `finalize()`'s prologue under `stateLock` and passed BY
168
+ // VALUE into the workQueue closure so the closure can capture
169
+ // `[payload, completion]` ONLY — zero `self` references — closing
170
+ // the entire class of `objc_retain`-on-torn-pointer crashes that
171
+ // fix-attempts 1-7 chased symptom-by-symptom on the same code path.
172
+ //
173
+ // Per design doc 2026-05-12-finalize-crash-investigation.md (C2
174
+ // escalation): all prior fixes attacked specific symptoms (which
175
+ // ivar's read raced, which lock's discipline was lax). This is the
176
+ // architectural escalation: by construction the closure has no
177
+ // access to mutable stitcher state, so no race in this code path is
178
+ // even possible.
179
+ //
180
+ // File-scope `internal` (not private) so future ports (Design 2
181
+ // actor, Design 3 C++) can adopt the same payload as their input.
182
+ //
183
+ // MAINTENANCE INVARIANT: every ivar finalize closures currently
184
+ // read (or might read in future edits) MUST live here. If you add
185
+ // a new finalize-relevant ivar to IncrementalStitcher,
186
+ // thread it through this struct. The CI test at
187
+ // scripts/check_c2_invariant.sh prevents accidental `self.*`
188
+ // reintroduction inside the closure body.
189
+ struct FinalizePayload {
190
+ // ── Output destination + quality ─────────────────────────────
191
+ /// The caller-supplied panorama output path, normalized
192
+ /// (file:// stripped if present).
193
+ let cleaned: String
194
+ /// JPEG quality, clamped to 1...100.
195
+ let q: Int
196
+
197
+ // ── Stitcher mode selection ─────────────────────────────────
198
+ /// True if this finalize is the V16 batch-keyframe pipeline.
199
+ let inBatchKeyframeMode: Bool
200
+ /// Hybrid engine ref (V14/V15 path). nil if batch mode.
201
+ let hybrid: OpenCVIncrementalStitcher?
202
+ /// First-wins cylindrical engine ref (V13 path). nil if batch mode.
203
+ let slit: OpenCVFirstWinsCylindricalStitcher?
204
+ /// V16 keyframe collector — owns the per-session JPEG sidecar
205
+ /// directory the post-stitch result references.
206
+ let collector: OpenCVKeyframeCollector?
207
+
208
+ // ── Frame inputs ─────────────────────────────────────────────
209
+ /// Absolute paths to keyframe JPEGs. Value-copied; the
210
+ /// underlying String storage is COW and immutable.
211
+ let paths: [String]
212
+
213
+ // ── cv::Stitcher tuning snapshots (fix4 lineage) ─────────────
214
+ let batchWarperType: String
215
+ let batchBlenderType: String
216
+ let batchSeamFinderType: String
217
+ let batchEnableInscribedRectCrop: Bool
218
+ let keyframeExifOrientation: Int
219
+ /// AR-STITCHING-TWO-MODES (memory/ar-stitching-two-modes.md):
220
+ /// capture-time hold orientation for the bake-rotation pass.
221
+ let captureOrientation: String
222
+
223
+ // ── Result metadata ──────────────────────────────────────────
224
+ /// Backpressure drops accumulated this capture, surfaced to JS.
225
+ let drops: Int
226
+ /// Whether the AR session was running at finalize-entry — drives
227
+ /// the AR restart in the closure's defer.
228
+ let arWasRunning: Bool
229
+ }
230
+
231
+
232
+ @objc(IncrementalStitcher)
233
+ public final class IncrementalStitcher: NSObject {
234
+
235
+ /// V13.0c.1.1 — same os_log subsystem as the slit-scan engine's
236
+ /// SlitDiagLog so Console.app sees both V13.0b-gate and V13.0c-trans
237
+ /// under one filter. FAULT-level survives NSLog's burst rate-limit
238
+ /// (~10/sec) — diagnostic logs at 50fps would otherwise be dropped.
239
+ fileprivate static let diagLog = OSLog(
240
+ subsystem: "com.tiger.retailens.sdk",
241
+ category: "slitscan"
242
+ )
243
+
244
+
245
+ @objc public static let shared = IncrementalStitcher()
246
+
247
+ /// Underlying OpenCV engine. Created on `start`, torn down on
248
+ /// `finalize`/`reset`. Holding it across captures would keep the
249
+ /// 24 MB canvas allocated in idle.
250
+ ///
251
+ /// V10: two engine variants exist behind one Swift wrapper.
252
+ /// `hybridEngine` (Samsung-style, full-frame cylindrical + OF) is
253
+ /// the default. `firstwinsEngine` (Apple-style, per-strip painting)
254
+ /// is opt-in via the JS `engine: 'slitscan'` start option. Only
255
+ /// one is non-nil at a time.
256
+ private var hybridEngine: OpenCVIncrementalStitcher?
257
+ private var firstwinsEngine: OpenCVFirstWinsCylindricalStitcher?
258
+
259
+ /// V15.0b — true once we've forwarded the latched plane transform
260
+ /// from RNSARSession to the slit-scan engine. Reset on
261
+ /// every start() so the next capture re-propagates. We only
262
+ /// forward once per capture: the plane transform is latched
263
+ /// (RNSARSession ignores subsequent ARKit refinements),
264
+ /// so re-propagating each frame is wasted work.
265
+ private var havePropagatedPlane: Bool = false
266
+
267
+ /// Convenience: read the active engine's accepted count. Used by
268
+ /// the per-frame state event.
269
+ private var engineAcceptedCount: Int {
270
+ return hybridEngine?.acceptedCount ?? firstwinsEngine?.acceptedCount ?? 0
271
+ }
272
+ private var anyEngineActive: Bool {
273
+ return hybridEngine != nil || firstwinsEngine != nil
274
+ }
275
+
276
+ /// Serial queue for the heavy per-frame work. ARSession delegate
277
+ /// only dispatches a pre-allocated cv::Mat onto this queue — the
278
+ /// pixel buffer itself is consumed before return.
279
+ ///
280
+ /// 2026-05-15 (C3 deferral) — investigated splitting the batch-
281
+ /// keyframe stitch onto its own DispatchQueue so a slow stitch
282
+ /// doesn't block ARSession frame ingestion on this workQueue.
283
+ /// Backed out: the existing `workQueue.sync` finalize boundary
284
+ /// (V16 Phase 1b.fix6) is INTENTIONAL — it serialises finalize
285
+ /// against in-flight frame work to prevent state-event races.
286
+ /// Moving the stitch to a separate queue requires reworking the
287
+ /// completion handler + stateLock contract to provide the same
288
+ /// ordering guarantees the sync barrier currently gives. Real
289
+ /// fix is non-trivial; deferred until pose-driven stitch work
290
+ /// lands (which will rework the queue topology anyway).
291
+ private let workQueue = DispatchQueue(
292
+ label: "com.retailens.incremental.stitcher",
293
+ qos: .userInitiated
294
+ )
295
+
296
+ /// 2026-05-16 — realtime+batch fusion (Option A "Replace on
297
+ /// completion"). Dedicated queue for the async refinement run
298
+ /// that follows a hybrid-engine finalize(). Kept SEPARATE from
299
+ /// `workQueue` so the next capture's start/consumeFrame path
300
+ /// isn't gated on the prior capture's 2-5 s cv::Stitcher run —
301
+ /// the design doc explicitly calls out "operator can continue
302
+ /// browsing / starting another capture during refinement".
303
+ ///
304
+ /// Serial: at most one refinement runs at a time (the design's
305
+ /// "cancellation semantics if a new capture starts mid-refine"
306
+ /// is out of scope for this MVP — see prompt's "deliberately out
307
+ /// of scope" list).
308
+ private let refineQueue = DispatchQueue(
309
+ label: "com.retailens.incremental.refine",
310
+ qos: .utility
311
+ )
312
+
313
+ /// Lock guarding `engine`/`isRunning` reads/writes. ARSession
314
+ /// delegate uses `try` to avoid blocking ARKit; if start/stop is
315
+ /// mid-flight the frame is dropped.
316
+ private let stateLock = NSLock()
317
+
318
+ /// Whether the engine is currently active. Set by start/stop.
319
+ @objc public private(set) var isRunning: Bool = false
320
+
321
+ /// The most recent state snapshot — readable by JS via the
322
+ /// bridge's `getState`.
323
+ private var lastState: IncrementalStateObject?
324
+
325
+ /// Cumulative drop counter — frames the work queue couldn't keep
326
+ /// up with. Diagnostic only; not surfaced to JS.
327
+ private var droppedBackpressure: Int = 0
328
+
329
+ /// V11 Gap #27: true when an ingest is in flight on the work
330
+ /// queue. Subsequent AR delegate frames are dropped (rather
331
+ /// than queued) so latency between AR time and canvas state
332
+ /// stays bounded.
333
+ private var workInFlight: Bool = false
334
+
335
+ /// Snapshot quality — host can pass on start.
336
+ private var snapshotJpegQuality: Int = 75
337
+
338
+ /// Periodic snapshot cadence — emit a panoramaPath update at most
339
+ /// every N accepts. Default 1 (every accept) gives the freshest
340
+ /// preview; field testing may show batching is friendlier on
341
+ /// the JS image-cache.
342
+ private var snapshotEveryNAccepts: Int = 1
343
+
344
+ /// Internal counter used with snapshotEveryNAccepts.
345
+ private var acceptsSinceSnapshot: Int = 0
346
+
347
+ /// V13.0c.1 — diagnostic state: first-frame world translation.
348
+ /// Used to compute Δtranslation per frame for the
349
+ /// [V13.0c-trans] log. We need to know how much users actually
350
+ /// translate during typical captures before committing to the
351
+ /// per-pixel depth correction architecture (V13.0c.2-.4).
352
+ /// Reset on each new capture (handled in `start()` below).
353
+ private var firstFrameTx: Double = 0
354
+ private var firstFrameTy: Double = 0
355
+ private var firstFrameTz: Double = 0
356
+ private var hasFirstFrameTranslation: Bool = false
357
+ private var consumeFrameCounter: Int = 0
358
+
359
+ /// V16 — pose-driven keyframe gate. When `enabled` (set from the
360
+ /// JS `frameSelectionMode = "pose-based"` config), each ARFrame is
361
+ /// projected onto the latched ARKit plane and accepted only when
362
+ /// it has ≥ `overlapThreshold` of NEW content vs the last
363
+ /// accepted keyframe. Bounded to `maxCount` keyframes per
364
+ /// capture. See KeyframeGate.swift for the full rationale.
365
+ private let keyframeGate = KeyframeGate()
366
+
367
+ /// V16 Phase 1 — when `engineMode == "batch-keyframe"`, no
368
+ /// incremental engine runs; we accumulate the gate-accepted
369
+ /// frames as on-disk JPEGs + their poses, then on `finalize` hand
370
+ /// them to `OpenCVStitcher.stitchKeyframePaths:withPoses:` (the
371
+ /// full BA + ExposureCompensator + GraphCutSeamFinder +
372
+ /// MultiBandBlender pipeline) for one-shot stitching. Why this
373
+ /// is structurally different from the slit-scan / hybrid engines:
374
+ /// they ingest into a streaming canvas, whereas batch-keyframe
375
+ /// defers all stitching until shutter release so the global-
376
+ /// stage quality wins (BA, multi-band) become available.
377
+ private var batchKeyframeMode: Bool = false
378
+ private var keyframeCollector: OpenCVKeyframeCollector?
379
+ /// Poses recorded 1:1 with `keyframeCollector`'s saved JPEGs.
380
+ /// Each entry is `RNSARFramePose.asDictionary()`. Reset
381
+ /// on every `start()`.
382
+ private var keyframePoses: [[String: Any]] = []
383
+ /// Saved JPEG paths in capture order. Tracked separately from
384
+ /// the collector so finalize doesn't have to reach back into ObjC.
385
+ private var keyframePaths: [String] = []
386
+ /// Frame rotation degrees passed at `start()` — needed when
387
+ /// saving keyframes so the JPEGs land in user-pan orientation
388
+ /// (the stitcher reads them in that orientation).
389
+ private var keyframeRotationDegrees: Int = 90
390
+ /// V16 Phase 1.fix2 — EXIF Orientation tag (1..8) baked into
391
+ /// each saved keyframe JPEG so iOS image renderers display
392
+ /// correctly while the stitcher (with IMREAD_IGNORE_ORIENTATION)
393
+ /// gets raw landscape pixels matching the pose's intrinsics.
394
+ private var keyframeExifOrientation: Int = 1
395
+ /// V16 Phase 1.fix4 — cv::Stitcher knobs. Defaults match the
396
+ /// modal's defaults and the legacy batch path defaults
397
+ /// (PlaneWarper + MultiBandBlender + GraphCutSeamFinder) since
398
+ /// fix4 routes through cv::Stitcher's feature-matched pipeline
399
+ /// where these defaults are the production-tested combo. User
400
+ /// can override via the modal Projection / Blender / Seam-finder
401
+ /// sections — those values flow through configOverrides at
402
+ /// start().
403
+ private var batchWarperType: String = "plane"
404
+ private var batchBlenderType: String = "multiband"
405
+ private var batchSeamFinderType: String = "graphcut"
406
+ // V16 Phase 1b.fix5c — operator-visible toggle for the
407
+ // max-inscribed-rectangle crop in the batch-keyframe finalize.
408
+ // Default OFF. When OFF, native crops to bbox only; when ON,
409
+ // the inscribed-rect + morph-close + col-projection pipeline
410
+ // runs.
411
+ private var batchEnableInscribedRectCrop: Bool = false
412
+ /// AR-STITCHING-TWO-MODES — see memory/ar-stitching-two-modes.md
413
+ ///
414
+ /// Physical phone orientation at start() time, sourced from the
415
+ /// JS accelerometer hook. Used to drive the OUTPUT panorama's
416
+ /// bake-rotation in OpenCVStitcher.stitchFramePaths. Held for
417
+ /// the lifetime of the capture so finalize() doesn't need to
418
+ /// re-sample (the user may have rotated the phone mid-pan and
419
+ /// we want the rotation that was correct WHEN they started).
420
+ ///
421
+ /// Valid values mirror IncrementalStartOptions.captureOrientation
422
+ /// in the JS API: "portrait", "portrait-upside-down",
423
+ /// "landscape-left", "landscape-right". Any other string is
424
+ /// treated as "portrait" (no rotation) by the .mm side.
425
+ private var captureOrientation: String = "portrait"
426
+
427
+ private override init() {
428
+ super.init()
429
+ }
430
+
431
+ /// 2026-05-18 (iOS cross-orientation fix) — bridge entry-point
432
+ /// that the bridge calls in finalize() when JS supplies a fresh
433
+ /// orientation. Overrides whatever start() snapshotted
434
+ /// (native ARKit query OR JS fallback). Used to close the bug
435
+ /// where the user opens the screen in orientation A, captures
436
+ /// in orientation B, and the bake_rotation table runs against
437
+ /// orientation A (the start-time value).
438
+ ///
439
+ /// Caller responsibility: this should only be called BEFORE
440
+ /// finalize() begins the stitch. Calling concurrently with an
441
+ /// in-flight stitch is a race on `self.captureOrientation`
442
+ /// (which the stitcher reads through the payload snapshot at
443
+ /// finalize() prologue). The bridge enforces "update then
444
+ /// finalize" sequentially on the workQueue.
445
+ @objc public func updateCaptureOrientation(_ orientation: String) {
446
+ stateLock.lock()
447
+ let prev = self.captureOrientation
448
+ self.captureOrientation = orientation
449
+ stateLock.unlock()
450
+ os_log(.fault, log: Self.diagLog,
451
+ "[V16-orchestrator] updateCaptureOrientation: %{public}@ → %{public}@",
452
+ prev, orientation)
453
+ }
454
+
455
+ /// 2026-05-18 (Iss 3) — return the current capture's keyframe
456
+ /// session directory, or nil if no capture is in flight / engine
457
+ /// isn't using a per-session keyframe collector.
458
+ @objc public func currentKeyframeDir() -> String? {
459
+ stateLock.lock()
460
+ defer { stateLock.unlock() }
461
+ guard self.isRunning, self.batchKeyframeMode else { return nil }
462
+ return self.keyframeCollector?.sessionDir
463
+ }
464
+
465
+ /// 2026-05-18 (Iss 3) — GC stale keyframe-session directories.
466
+ ///
467
+ /// Scans `Library/Application Support/Captures/` for subdirectories
468
+ /// whose newest file mtime is older than `cutoffMs` (ms past epoch).
469
+ /// Each stale subdirectory is removed in full (it's a session UUID
470
+ /// dir with N keyframe JPEGs). Sessions whose newest file is newer
471
+ /// than the cutoff are LEFT ALONE — even if they look stale by some
472
+ /// other heuristic — because they might belong to a capture that's
473
+ /// still in flight (the engine writes incremental frames to the
474
+ /// same dir).
475
+ ///
476
+ /// Returns a tuple of (sessionsDeleted, bytesFreed) so the bridge
477
+ /// can surface those numbers to JS for an optional UX toast. All
478
+ /// filesystem errors are swallowed and counted as "not deleted";
479
+ /// host should NEVER see a thrown error from this — at worst it
480
+ /// gets back zero counts and can investigate Console.app logs.
481
+ @objc public func cleanupKeyframes(
482
+ olderThanMs: Double
483
+ ) -> [String: NSNumber] {
484
+ let cutoff = Date().timeIntervalSince1970 - (olderThanMs / 1000.0)
485
+ let fm = FileManager.default
486
+ guard let appSupport = try? fm.url(
487
+ for: .applicationSupportDirectory,
488
+ in: .userDomainMask,
489
+ appropriateFor: nil,
490
+ create: false // don't create if missing — that means nothing to clean
491
+ ) else {
492
+ return ["sessionsDeleted": 0, "bytesFreed": 0]
493
+ }
494
+ let capturesURL = appSupport.appendingPathComponent("Captures", isDirectory: true)
495
+ guard fm.fileExists(atPath: capturesURL.path) else {
496
+ return ["sessionsDeleted": 0, "bytesFreed": 0]
497
+ }
498
+ guard let sessions = try? fm.contentsOfDirectory(
499
+ at: capturesURL,
500
+ includingPropertiesForKeys: [.contentModificationDateKey, .isDirectoryKey, .fileSizeKey],
501
+ options: [.skipsHiddenFiles]
502
+ ) else {
503
+ return ["sessionsDeleted": 0, "bytesFreed": 0]
504
+ }
505
+
506
+ var sessionsDeleted = 0
507
+ var bytesFreed: UInt64 = 0
508
+ for sessionURL in sessions {
509
+ // Only dirs are real sessions; skip stray files.
510
+ let isDir = (try? sessionURL.resourceValues(
511
+ forKeys: [.isDirectoryKey]
512
+ ))?.isDirectory ?? false
513
+ if !isDir { continue }
514
+ // Newest mtime across the session's files (recursive — though
515
+ // the collector writes a flat dir today, future-proof).
516
+ var newestMtime: TimeInterval = 0
517
+ var sessionBytes: UInt64 = 0
518
+ if let enumerator = fm.enumerator(
519
+ at: sessionURL,
520
+ includingPropertiesForKeys: [.contentModificationDateKey, .fileSizeKey],
521
+ options: [.skipsHiddenFiles]
522
+ ) {
523
+ for case let fileURL as URL in enumerator {
524
+ let r = try? fileURL.resourceValues(
525
+ forKeys: [.contentModificationDateKey, .fileSizeKey]
526
+ )
527
+ if let mtime = r?.contentModificationDate?.timeIntervalSince1970 {
528
+ if mtime > newestMtime { newestMtime = mtime }
529
+ }
530
+ if let bytes = r?.fileSize {
531
+ sessionBytes += UInt64(bytes)
532
+ }
533
+ }
534
+ }
535
+ // Use the directory's own mtime as a fallback if no files
536
+ // matched (an empty session dir is also stale).
537
+ if newestMtime == 0 {
538
+ if let dirMtime = (try? sessionURL.resourceValues(
539
+ forKeys: [.contentModificationDateKey]
540
+ ))?.contentModificationDate?.timeIntervalSince1970 {
541
+ newestMtime = dirMtime
542
+ }
543
+ }
544
+ if newestMtime > 0 && newestMtime < cutoff {
545
+ if (try? fm.removeItem(at: sessionURL)) != nil {
546
+ sessionsDeleted += 1
547
+ bytesFreed += sessionBytes
548
+ }
549
+ }
550
+ }
551
+ os_log(.fault, log: Self.diagLog,
552
+ "[V16-orchestrator] cleanupKeyframes olderThanMs=%.0f sessions=%d bytes=%llu",
553
+ olderThanMs, Int32(sessionsDeleted), bytesFreed)
554
+ return [
555
+ "sessionsDeleted": NSNumber(value: sessionsDeleted),
556
+ "bytesFreed": NSNumber(value: bytesFreed),
557
+ ]
558
+ }
559
+
560
+ /// 2026-05-16 — realtime+batch fusion (Option A) path derivation.
561
+ /// Given the live panorama path (which finalize() wrote inside
562
+ /// the app sandbox tmp or a host-supplied location), pick a path
563
+ /// for the refined output. Pattern:
564
+ ///
565
+ /// /…/RNImageStitcherIncremental-<uuid>.jpg
566
+ /// → /…/RNImageStitcherIncremental-<uuid>-refined.jpg
567
+ ///
568
+ /// Same directory keeps cleanup discoverable (delete both when
569
+ /// the audit is discarded). Different name avoids racing the
570
+ /// host UI that may still be reading the live file as the
571
+ /// refinement is writing.
572
+ fileprivate static func refinedPathFromLive(livePath: String) -> String {
573
+ let ns = livePath as NSString
574
+ let dir = ns.deletingLastPathComponent
575
+ let base = (ns.lastPathComponent as NSString).deletingPathExtension
576
+ let ext = (ns.lastPathComponent as NSString).pathExtension
577
+ let refinedName = ext.isEmpty
578
+ ? "\(base)-refined"
579
+ : "\(base)-refined.\(ext)"
580
+ return (dir as NSString).appendingPathComponent(refinedName)
581
+ }
582
+
583
+ // ── Native orientation classifier ────────────────────────────────
584
+ //
585
+ // AR-STITCHING-TWO-MODES — see memory/ar-stitching-two-modes.md
586
+ //
587
+ // Why this exists: JS's useDeviceOrientation hook ships React
588
+ // state that's stale at the moment incremental.start() is invoked
589
+ // (the hook samples accelerometer at 10 Hz and renders through
590
+ // React's normal update cycle — by the time the start() callback
591
+ // runs, the closure has captured the previous-state value).
592
+ // Field test on 2026-05-11 confirmed: 3/3 captures (Mode B,
593
+ // Mode A landscape-left, Mode A landscape-right) all arrived at
594
+ // the bridge with captureOrientation="portrait" even though the
595
+ // user explicitly rotated to landscape for two of them.
596
+ //
597
+ // ARKit gives us the answer directly: `frame.camera.eulerAngles.z`
598
+ // is the camera's roll (rotation around its optical axis) — the
599
+ // exact value that distinguishes portrait / landscape-left /
600
+ // landscape-right / portrait-upside-down regardless of any UI
601
+ // lock or JS state staleness. Read it synchronously at start()
602
+ // time and use it as the source of truth.
603
+ //
604
+ // Convention (verified against ARKit docs + iOS device axes):
605
+ // • Camera local axes: +X right, +Y up (along phone length
606
+ // when held portrait), +Z back (toward user's face).
607
+ // • Roll = rotation around camera's +Z (right-hand rule).
608
+ // • Portrait → roll ≈ 0
609
+ // • Landscape-left (CCW) → roll ≈ +π/2 (verified empirically;
610
+ // swap with -right if first test
611
+ // shows landscape-right hitting
612
+ // this branch instead)
613
+ // • Portrait-upside-down → roll ≈ ±π
614
+ // • Landscape-right (CW) → roll ≈ -π/2
615
+ //
616
+ // ±45° tolerance per quadrant keeps classification stable under
617
+ // small hand wobble.
618
+ private static func nativeCaptureOrientation() -> (
619
+ orientation: String,
620
+ rollDegrees: Double,
621
+ hadFrame: Bool
622
+ ) {
623
+ guard let frame = RNSARSession.shared.arSession.currentFrame else {
624
+ // No AR frame yet — falls back to "portrait" (Mode B start
625
+ // state). Should be rare: incremental.start() requires
626
+ // ARSession to be running, which means frames are flowing.
627
+ return ("portrait", 0.0, false)
628
+ }
629
+ let rollRadians = Double(frame.camera.eulerAngles.z)
630
+ let rollDegrees = rollRadians * 180.0 / .pi
631
+ let classified: String
632
+ // Empirically calibrated against Ram's 2026-05-11 3-capture
633
+ // test (1st=L-left, 2nd=portrait, 3rd=L-right):
634
+ // Ram's "landscape-left" → roll ≈ 0° (NOT -90° as I assumed)
635
+ // Ram's "portrait" → roll ≈ -90°
636
+ // Ram's "landscape-right" → roll ≈ ±180°
637
+ // Ram's "portrait-upside-down" → roll ≈ +90° (by symmetry, untested)
638
+ //
639
+ // Why this differs from the device-orientation intuition:
640
+ // ARKit's `camera.eulerAngles.z` is the camera's roll around
641
+ // its optical axis, measured against world-up. The iPhone's
642
+ // image sensor is mounted such that its long axis aligns
643
+ // with the phone's WIDTH (not length), so the camera's +Y
644
+ // is naturally aligned with world-UP when the phone is in
645
+ // landscape — roll = 0 means the phone is landscape.
646
+ // Holding the phone in portrait rotates the camera +Y to
647
+ // horizontal in world → roll = ±90°.
648
+ //
649
+ // The rotation table in OpenCVStitcher.mm (landscape-left →
650
+ // 90° CCW, portrait → none, etc.) is unchanged — only the
651
+ // classifier mapping is shifted by 90°.
652
+ if abs(rollDegrees) < 45 {
653
+ classified = "landscape-left"
654
+ } else if rollDegrees >= 45 && rollDegrees < 135 {
655
+ classified = "portrait-upside-down"
656
+ } else if rollDegrees <= -45 && rollDegrees > -135 {
657
+ classified = "portrait"
658
+ } else {
659
+ classified = "landscape-right"
660
+ }
661
+ return (classified, rollDegrees, true)
662
+ }
663
+
664
+ // ── Public lifecycle ────────────────────────────────────────────
665
+
666
+ /// Begin a new incremental capture. Hooks the ARSession's
667
+ /// per-frame stream into the engine. Caller must already have
668
+ /// the AR session running (start/stop is the host app's job).
669
+ @objc public func start(
670
+ composeWidth: Int,
671
+ composeHeight: Int,
672
+ canvasWidth: Int,
673
+ canvasHeight: Int,
674
+ featherPx: Int,
675
+ snapshotJpegQuality: Int,
676
+ snapshotEveryNAccepts: Int,
677
+ frameRotationDegrees: Int,
678
+ engineMode: String,
679
+ captureOrientation: String = "portrait",
680
+ configOverrides: [String: Any] = [:],
681
+ // 2026-05-18 (Issue #2 regression fix): "arSession" (default,
682
+ // legacy) registers as the ARSession's frame consumer.
683
+ // "jsDriver" skips that registration — frames will come in
684
+ // via processFrameAtPath instead. Used by iOS non-AR
685
+ // captures (the vision-camera + gyro driver path).
686
+ frameSourceMode: String = "arSession"
687
+ ) {
688
+ stateLock.lock()
689
+ if isRunning {
690
+ stateLock.unlock()
691
+ return
692
+ }
693
+ // AR-STITCHING-TWO-MODES — see memory/ar-stitching-two-modes.md
694
+ // Override the JS-supplied captureOrientation with a native
695
+ // ARKit reading. Field test on 2026-05-11 confirmed JS state
696
+ // is consistently stale at start() time (always "portrait"
697
+ // regardless of physical orientation); native AR frame data
698
+ // is real-time and unambiguous. See nativeCaptureOrientation
699
+ // for the full RCA. JS-supplied value retained in the log
700
+ // for diagnostic purposes.
701
+ let nativeResult = Self.nativeCaptureOrientation()
702
+ let resolvedOrientation = nativeResult.hadFrame
703
+ ? nativeResult.orientation
704
+ : captureOrientation // no AR frame yet — fall back to JS
705
+ self.captureOrientation = resolvedOrientation
706
+ os_log(.fault, log: Self.diagLog,
707
+ "[V16-orchestrator] start: JS sent=%{public}@ native roll=%.1f° → resolved=%{public}@ (native_used=%d) engineMode=%{public}@",
708
+ captureOrientation,
709
+ nativeResult.rollDegrees,
710
+ resolvedOrientation,
711
+ nativeResult.hadFrame ? Int32(1) : Int32(0),
712
+ engineMode)
713
+ // V15 — engine modes:
714
+ // 'hybrid' → hybrid engine, planar projection by default
715
+ // 'slitscan-rotate' → slit-scan, rectilinear, V13.0a + 1D NCC
716
+ // 'slitscan-both' → slit-scan, rectilinear, V13.0a + no gate
717
+ // + feather blend (iterate via overrides)
718
+ // Backward compat in -[RLISStitcherConfig configForMode:] handles
719
+ // 'firstwins-rectilinear' → 'slitscan-rotate' and warns on
720
+ // legacy 'firstwins' / 'firstwins-zoomed' / 'slitscan' modes.
721
+ let normalisedMode: String
722
+ switch engineMode {
723
+ case "hybrid": normalisedMode = "hybrid"
724
+ case "batch-keyframe":
725
+ // V16 Phase 1 — new mode. Skips the live incremental
726
+ // engines entirely; KeyframeGate accumulates accepted
727
+ // frames as JPEGs, on finalize OpenCVStitcher does the
728
+ // full-pipeline stitch.
729
+ normalisedMode = "batch-keyframe"
730
+ case "slitscan-rotate", "firstwins-rectilinear":
731
+ normalisedMode = "slitscan-rotate"
732
+ case "slitscan-both":
733
+ normalisedMode = "slitscan-both"
734
+ case "firstwins", "firstwins-zoomed", "slitscan":
735
+ NSLog("[V15-bridge] DEPRECATED engine '\(engineMode)' — using slitscan-both")
736
+ normalisedMode = "slitscan-both"
737
+ default:
738
+ NSLog("[V15-bridge] unknown engine '\(engineMode)' — using slitscan-both")
739
+ normalisedMode = "slitscan-both"
740
+ }
741
+
742
+ let useBatchKeyframe = (normalisedMode == "batch-keyframe")
743
+ let useFirstwinsClass = normalisedMode.hasPrefix("slitscan")
744
+
745
+ // Build the V15 config: factory default for the mode, then apply
746
+ // JS-side overrides.
747
+ let config = RLISStitcherConfig(forMode: normalisedMode)
748
+ Self.applyConfigOverrides(configOverrides, to: config)
749
+
750
+ if useBatchKeyframe {
751
+ // V16 Phase 1 — no live engine; spin up a keyframe
752
+ // collector that saves accepted frames to disk under
753
+ // Library/AppSupport/Captures/{uuid}/. On finalize
754
+ // these are handed to OpenCVStitcher's full pipeline
755
+ // (BA + GraphCut + ExposureComp + MultiBand) — the
756
+ // actually-quality path. Memory is bounded because
757
+ // KeyframeGate caps input at `keyframeMaxCount` (≤6).
758
+ do {
759
+ self.keyframeCollector = try OpenCVKeyframeCollector()
760
+ } catch {
761
+ NSLog("[V16-batch-keyframe] collector init failed: \(error.localizedDescription)")
762
+ self.keyframeCollector = nil
763
+ }
764
+ self.keyframePaths = []
765
+ self.keyframePoses = []
766
+ // V16 Phase 1.fix1 — keep frames in native landscape
767
+ // sensor orientation for the batch-keyframe path so the
768
+ // pose's intrinsics (which describe the unrotated
769
+ // 1920×1440 sensor) match the saved-image dimensions.
770
+ // The slit-scan and hybrid engines continue to receive
771
+ // `frameRotationDegrees` unchanged.
772
+ self.keyframeRotationDegrees = 0
773
+ // 2026-05-18 (Issue #1a fix) — keyframe EXIF Orientation
774
+ // is hardcoded to 6 ("rotate 90° CW for display") regardless
775
+ // of physical capture orientation.
776
+ //
777
+ // RCA: the earlier V16 Phase 1.fix2 mapping branched on
778
+ // `frameRotationDegrees` to "encode the user's perceived
779
+ // orientation". That was written for a hypothetical
780
+ // app whose orientation locks WITH the user's hold. Our
781
+ // host app is PORTRAIT-LOCKED: the rendering surface
782
+ // never rotates, regardless of how the operator holds
783
+ // the phone.
784
+ //
785
+ // RN's <Image> + Files.app honour EXIF when rendering.
786
+ // Sensor-native pixels are landscape-aspect (long axis =
787
+ // phone-Y). In a portrait-locked UI, displaying with
788
+ // EXIF=1 leaves the JPEG in landscape pixels rendered
789
+ // INTO portrait UI — the operator, head tilted to view
790
+ // their landscape capture, then sees the band thumbnails
791
+ // rotated 90° from their world view ("sideways"). Pre-
792
+ // bug-fix, the broken useDeviceOrientation hook always
793
+ // reported 'portrait' so `frameRotationDegrees=90` was
794
+ // always selected and EXIF=6 was always written — the
795
+ // operator never saw the misalignment because EXIF=6's
796
+ // 90° CW display rotation cancelled their physical 90°
797
+ // CCW head tilt in landscape-left view. Fixing the hook
798
+ // exposed the underlying portrait-lock mismatch.
799
+ //
800
+ // EXIF=6 (always) keeps the band thumbnails consistent
801
+ // with the portrait-locked UI in every hold. The FINAL
802
+ // panorama bake is independent — it consumes
803
+ // `config.captureOrientation` in cpp/stitcher.cpp's
804
+ // bake_rotation pass and is unaffected by this constant.
805
+ //
806
+ // If we ever unlock the host app's orientation (so the
807
+ // UI rotates with the user), this should revert to the
808
+ // 4-way switch. Tracked as a follow-up in the design
809
+ // doc.
810
+ self.keyframeExifOrientation = 6
811
+ // V16 Phase 1.fix4 — read cv::Stitcher knobs from JS config.
812
+ // Defaults to "plane" / "multiband" / "graphcut" — the
813
+ // proven combo cv::Stitcher::PANORAMA uses internally.
814
+ // Operator can A/B different warpers from the modal's
815
+ // Projection / Blender / Seam-finder sections.
816
+ self.batchWarperType =
817
+ (configOverrides["warperType"] as? String) ?? "plane"
818
+ self.batchBlenderType =
819
+ (configOverrides["blenderType"] as? String) ?? "multiband"
820
+ self.batchSeamFinderType =
821
+ (configOverrides["seamFinderType"] as? String) ?? "graphcut"
822
+ // V16 Phase 1b.fix5c — read inscribed-rect toggle. Defaults
823
+ // to FALSE if not provided by JS.
824
+ self.batchEnableInscribedRectCrop =
825
+ (configOverrides["enableMaxInscribedRectCrop"] as? Bool) ?? false
826
+ self.batchKeyframeMode = true
827
+ self.hybridEngine = nil
828
+ self.firstwinsEngine = nil
829
+ os_log(.fault, log: Self.diagLog,
830
+ "[V16-batch-keyframe] start mode=batch-keyframe rotation=0 (was %d, forced to 0 to match pose intrinsics) sessionDir=%{public}@",
831
+ frameRotationDegrees,
832
+ self.keyframeCollector?.sessionDir ?? "(nil)")
833
+ } else if useFirstwinsClass {
834
+ // Slit-scan engine always uses rectilinear in V15
835
+ // (firstwins-cylindrical and firstwins-zoomed modes were
836
+ // removed; their behaviour is unused).
837
+ self.firstwinsEngine = OpenCVFirstWinsCylindricalStitcher(
838
+ composeWidth: composeWidth,
839
+ composeHeight: composeHeight,
840
+ canvasWidth: canvasWidth,
841
+ canvasHeight: canvasHeight,
842
+ featherPx: featherPx,
843
+ frameRotationDegrees: frameRotationDegrees,
844
+ useRectilinear: true
845
+ )
846
+ self.firstwinsEngine?.setConfig(config)
847
+ self.hybridEngine = nil
848
+ self.batchKeyframeMode = false
849
+ } else {
850
+ self.hybridEngine = OpenCVIncrementalStitcher(
851
+ composeWidth: composeWidth,
852
+ composeHeight: composeHeight,
853
+ canvasWidth: canvasWidth,
854
+ canvasHeight: canvasHeight,
855
+ featherPx: featherPx,
856
+ frameRotationDegrees: frameRotationDegrees
857
+ )
858
+ self.hybridEngine?.setConfig(config)
859
+ self.firstwinsEngine = nil
860
+ self.batchKeyframeMode = false
861
+ }
862
+ self.isRunning = true
863
+ self.snapshotJpegQuality = max(1, min(100, snapshotJpegQuality))
864
+ self.snapshotEveryNAccepts = max(1, snapshotEveryNAccepts)
865
+ self.acceptsSinceSnapshot = 0
866
+ self.droppedBackpressure = 0
867
+ self.lastState = nil
868
+ // V15.0b — re-arm plane propagation for the new capture.
869
+ self.havePropagatedPlane = false
870
+ // V13.0c.1 — reset translation diagnostic state for the
871
+ // new capture. First-frame translation will be captured
872
+ // on the next consumeFrame call.
873
+ self.hasFirstFrameTranslation = false
874
+ self.consumeFrameCounter = 0
875
+
876
+ // V16 — configure the pose-driven keyframe gate from JS
877
+ // config overrides. Defaults match the field-tested values
878
+ // for a 90° landscape pan over a retail shelf: 40% required
879
+ // new content per keyframe, capped at 6 keyframes per
880
+ // capture. Values out of range are clamped silently.
881
+ let frameMode = (configOverrides["frameSelectionMode"] as? String)
882
+ ?? "time-based"
883
+ // V16 A2 — both 'pose-based' and 'flow-based' enable the gate;
884
+ // they differ only in the novelty metric (plane-overlap vs
885
+ // sparse-flow). 'time-based' = passthrough.
886
+ self.keyframeGate.enabled =
887
+ (frameMode == "pose-based" || frameMode == "flow-based")
888
+ self.keyframeGate.strategy =
889
+ (frameMode == "flow-based") ? .flow : .pose
890
+ if let v = configOverrides["keyframeOverlapThreshold"] as? Double {
891
+ self.keyframeGate.overlapThreshold = max(0.10, min(0.80, v))
892
+ } else {
893
+ self.keyframeGate.overlapThreshold = 0.4
894
+ }
895
+ if let v = configOverrides["keyframeMaxCount"] as? Int {
896
+ self.keyframeGate.maxCount = max(3, min(10, v))
897
+ } else {
898
+ self.keyframeGate.maxCount = 6
899
+ }
900
+ // V16 A2 — flow tuning. C++ side also clamps defensively
901
+ // (setFlowMaxCorners ≥ 30, setFlowQualityLevel ∈ (0, 1],
902
+ // setFlowMinDistance ≥ 1.0) so we layer the JS-side modal
903
+ // ranges over those minimum invariants. Reading these
904
+ // unconditionally — they're cheap and the gate ignores them
905
+ // when strategy != .flow.
906
+ if let v = configOverrides["flowMaxCorners"] as? Int {
907
+ self.keyframeGate.flowMaxCorners = max(50, min(300, v))
908
+ } else {
909
+ self.keyframeGate.flowMaxCorners = 150
910
+ }
911
+ if let v = configOverrides["flowQualityLevel"] as? Double {
912
+ self.keyframeGate.flowQualityLevel = max(0.005, min(0.05, v))
913
+ } else {
914
+ self.keyframeGate.flowQualityLevel = 0.01
915
+ }
916
+ if let v = configOverrides["flowMinDistance"] as? Double {
917
+ self.keyframeGate.flowMinDistance = max(1.0, min(50.0, v))
918
+ } else if let v = configOverrides["flowMinDistance"] as? Int {
919
+ // JS sometimes ships Ints when the value happens to be
920
+ // an integer (SegmentedControl options are strings that
921
+ // parseInt into Ints). Accept either shape.
922
+ self.keyframeGate.flowMinDistance = max(1.0, min(50.0, Double(v)))
923
+ } else {
924
+ self.keyframeGate.flowMinDistance = 10.0
925
+ }
926
+ // V16 — translation-budget force-accept (Flow strategy). cm
927
+ // on the JS side (UI-friendly), converted to metres by the
928
+ // KeyframeGate.swift setter. Clamp to [0, 100] cm at start so
929
+ // a stray JS default can't put the gate in an unworkable
930
+ // state. Default 0 = disabled (preserves pre-V16 behaviour
931
+ // for callers that don't opt in).
932
+ if let v = configOverrides["flowMaxTranslationCm"] as? Double {
933
+ self.keyframeGate.flowMaxTranslationCm = max(0.0, min(100.0, v))
934
+ } else if let v = configOverrides["flowMaxTranslationCm"] as? Int {
935
+ self.keyframeGate.flowMaxTranslationCm = max(0.0, min(100.0, Double(v)))
936
+ } else {
937
+ self.keyframeGate.flowMaxTranslationCm = 0.0
938
+ }
939
+ // V16 — novelty aggregation percentile. Clamp at start to
940
+ // [0.5, 0.99]; the bridge re-clamps but matching it here
941
+ // means our state stays in-range for logging. Default 0.85
942
+ // — picks up leading-edge motion sooner than the pre-V16
943
+ // median (0.5).
944
+ if let v = configOverrides["flowNoveltyPercentile"] as? Double {
945
+ self.keyframeGate.flowNoveltyPercentile = max(0.5, min(0.99, v))
946
+ } else {
947
+ self.keyframeGate.flowNoveltyPercentile = 0.85
948
+ }
949
+ // V16 — Swift-side eval throttle. Default 1 (every consumeFrame
950
+ // runs evaluate). Range 1-10. At 1, identical to pre-V16
951
+ // behaviour; at higher N, evaluate runs every Nth frame to
952
+ // save CPU/battery on long captures. Doesn't change WHICH
953
+ // frames are accepted (still subject to overlapThreshold +
954
+ // translation budget) — just samples less frequently.
955
+ if let v = configOverrides["flowEvalEveryNFrames"] as? Int {
956
+ self.keyframeGate.flowEvalEveryNFrames = max(1, min(10, v))
957
+ } else {
958
+ self.keyframeGate.flowEvalEveryNFrames = 1
959
+ }
960
+ self.keyframeGate.reset()
961
+ os_log(.fault, log: Self.diagLog,
962
+ "[V16-keyframe] start gate enabled=%d strategy=%{public}@ thr=%.2f max=%d flow(maxCorners=%d quality=%.3f minDist=%.1f maxTransCm=%.1f pctile=%.2f evalEveryN=%d)",
963
+ self.keyframeGate.enabled ? 1 : 0,
964
+ self.keyframeGate.strategy == .flow ? "flow" : "pose",
965
+ self.keyframeGate.overlapThreshold,
966
+ self.keyframeGate.maxCount,
967
+ self.keyframeGate.flowMaxCorners,
968
+ self.keyframeGate.flowQualityLevel,
969
+ self.keyframeGate.flowMinDistance,
970
+ self.keyframeGate.flowMaxTranslationCm,
971
+ self.keyframeGate.flowNoveltyPercentile,
972
+ Int32(self.keyframeGate.flowEvalEveryNFrames))
973
+
974
+ stateLock.unlock()
975
+
976
+ // Register with the AR session — only when running in the
977
+ // AR-frame-stream-driven mode. In jsDriver mode (iOS non-AR
978
+ // captures) the AR session is intentionally stopped so the
979
+ // vision-camera holds the camera; frames arrive via
980
+ // processFrameAtPath from JS instead. Registering as the
981
+ // consumer here would either crash (no running session) or
982
+ // mis-route frames once an AR session somewhere else came up.
983
+ if frameSourceMode != "jsDriver" {
984
+ RNSARSession.shared.incrementalConsumer = self
985
+ }
986
+ }
987
+
988
+ /// Stop ingestion + write the final panorama to `outputPath`.
989
+ /// Returns the result on the main thread via completion.
990
+ ///
991
+ /// V12.1 frame-leak fix: the previous version waited until the
992
+ /// finalize block ran on the work queue to set isRunning=false.
993
+ /// Between the JS shutter-release and that block running, the
994
+ /// AR delegate could deliver several more frames — each one
995
+ /// passed consumeFrame's `isRunning == true` check and got
996
+ /// ingested into the canvas, producing visible "phantom" frames
997
+ /// V15 — apply per-stage JS overrides on top of a mode default.
998
+ /// Keys recognised in `overrides`: any non-readonly RLISStitcherConfig
999
+ /// field. Unrecognised keys are ignored. Values out of range are
1000
+ /// clamped silently (e.g. kPanAxisFractionRect outside [0.05, 0.90]).
1001
+ private static func applyConfigOverrides(_ overrides: [String: Any],
1002
+ to config: RLISStitcherConfig) {
1003
+ if let v = overrides["kPanAxisFractionRect"] as? Double {
1004
+ config.kPanAxisFractionRect = max(0.05, min(0.90, v))
1005
+ }
1006
+ if let v = overrides["kMinAcceptDeltaPx"] as? Int {
1007
+ config.kMinAcceptDeltaPx = max(0, min(500, v))
1008
+ }
1009
+ if let v = overrides["enableTriangulation"] as? Bool {
1010
+ config.enableTriangulation = v
1011
+ }
1012
+ if let v = overrides["enableTriAccumulator"] as? Bool {
1013
+ config.enableTriAccumulator = v
1014
+ }
1015
+ if let v = overrides["enable1dNcc"] as? Bool {
1016
+ config.enable1dNcc = v
1017
+ }
1018
+ if let v = overrides["nccSearchRadius1d"] as? Int {
1019
+ config.nccSearchRadius1d = max(5, min(60, v))
1020
+ }
1021
+ if let v = overrides["enable2dNcc"] as? Bool {
1022
+ config.enable2dNcc = v
1023
+ }
1024
+ if let v = overrides["enableRansacHomography"] as? Bool {
1025
+ config.enableRansacHomography = v
1026
+ }
1027
+ if let v = overrides["paintMode"] as? String {
1028
+ switch v {
1029
+ case "FirstPaintedWins": config.paintMode = .firstPaintedWins
1030
+ case "FeatherBlend": config.paintMode = .featherBlend
1031
+ default: break
1032
+ }
1033
+ }
1034
+ if let v = overrides["hybridProjection"] as? String {
1035
+ switch v {
1036
+ case "Cylindrical": config.hybridProjection = .cylindrical
1037
+ case "Planar": config.hybridProjection = .planar
1038
+ default: break
1039
+ }
1040
+ }
1041
+ if let v = overrides["useDetectedPlane"] as? Bool {
1042
+ config.useDetectedPlane = v
1043
+ }
1044
+ if let v = overrides["sliverPosition"] as? String {
1045
+ switch v {
1046
+ case "Center": config.sliverPosition = .center
1047
+ case "Bottom": config.sliverPosition = .bottom
1048
+ case "Top": config.sliverPosition = .top
1049
+ default: break
1050
+ }
1051
+ }
1052
+ if let v = overrides["firstFrameFullFrame"] as? Bool {
1053
+ config.firstFrameFullFrame = v
1054
+ }
1055
+ // V15.0d new overrides.
1056
+ if let v = overrides["planeSource"] as? String {
1057
+ switch v {
1058
+ case "Disabled": config.planeSource = .disabled
1059
+ case "ARKitDetected": config.planeSource = .arKitDetected
1060
+ case "Virtual": config.planeSource = .virtual
1061
+ default: break
1062
+ }
1063
+ }
1064
+ if let v = overrides["virtualPlaneDepthMeters"] as? Double {
1065
+ config.virtualPlaneDepthMeters = max(0.3, min(5.0, v))
1066
+ }
1067
+ if let v = overrides["arkitPlaneAlignmentThreshold"] as? Double {
1068
+ config.arkitPlaneAlignmentThreshold = max(0.0, min(1.0, v))
1069
+ }
1070
+ if let v = overrides["planeProjectionStyle"] as? String {
1071
+ switch v {
1072
+ case "Trapezoidal": config.planeProjectionStyle = .trapezoidal
1073
+ case "Rectified": config.planeProjectionStyle = .rectified
1074
+ default: break
1075
+ }
1076
+ }
1077
+ if let v = overrides["nccSearchMargin2d"] as? Int {
1078
+ config.nccSearchMargin2d = max(4, min(60, v))
1079
+ }
1080
+ if let v = overrides["nccConfidenceThreshold2d"] as? Double {
1081
+ config.nccConfidenceThreshold2d = max(0.30, min(0.99, v))
1082
+ }
1083
+ if let v = overrides["enableNcc2dEmaSmoothing"] as? Bool {
1084
+ config.enableNcc2dEmaSmoothing = v
1085
+ }
1086
+ if let v = overrides["ncc2dEmaAlpha"] as? Double {
1087
+ config.ncc2dEmaAlpha = max(0.05, min(0.95, v))
1088
+ }
1089
+ if let v = overrides["enableNcc2dPanAxisLock"] as? Bool {
1090
+ config.enableNcc2dPanAxisLock = v
1091
+ }
1092
+ if let v = overrides["ncc2dCrossAxisLockPx"] as? Int {
1093
+ config.ncc2dCrossAxisLockPx = max(0, min(30, v))
1094
+ }
1095
+ // Propagate the alignment threshold to the AR session so its
1096
+ // didAdd / didUpdate filter uses the operator-chosen value.
1097
+ // (planeAlignmentThreshold is a Float on the AR session.)
1098
+ RNSARSession.shared.planeAlignmentThreshold =
1099
+ Float(config.arkitPlaneAlignmentThreshold)
1100
+ }
1101
+
1102
+ /// after the user thought they had released. The engine refs
1103
+ /// and isRunning flag are now flipped SYNCHRONOUSLY here so the
1104
+ /// AR delegate's very next consumeFrame sees isRunning=false.
1105
+ /// The work-queue body just runs the engine's own finalize.
1106
+ @objc public func finalize(
1107
+ toPath outputPath: String,
1108
+ jpegQuality: Int,
1109
+ completion: @escaping ([String: Any]?, NSError?) -> Void
1110
+ ) {
1111
+ // V12.9 fix #3 — flip isRunning=false BEFORE nulling the AR
1112
+ // consumer. The previous order had a race window: AR
1113
+ // delegate captures `incrementalConsumer` (non-nil), the
1114
+ // delegate is briefly suspended, finalize runs and sets
1115
+ // consumer=nil + isRunning=false, then the suspended delegate
1116
+ // resumes and calls consumer.consumeFrame(). consumeFrame
1117
+ // saw isRunning=false at its first guard and bailed in MOST
1118
+ // cases, but the in-flight workQueue task (if any) had
1119
+ // already captured non-nil engine refs at consumeFrame
1120
+ // entry — by re-checking inside the workQueue async we
1121
+ // catch it, but only just. Flipping isRunning first
1122
+ // collapses the race: any consumeFrame entered after this
1123
+ // line sees isRunning=false at its very first guard.
1124
+ stateLock.lock()
1125
+ let hybrid = self.hybridEngine
1126
+ let slit = self.firstwinsEngine
1127
+ let inBatchKeyframeMode = self.batchKeyframeMode
1128
+ let collector = self.keyframeCollector
1129
+ let paths = self.keyframePaths
1130
+ // V16 Phase 1b.fix4 — snapshot the cv::Stitcher knobs and
1131
+ // EXIF orientation under stateLock so the workQueue closure
1132
+ // has a stable view of these values, independent of any
1133
+ // concurrent start() that may begin a new capture before
1134
+ // this closure finishes.
1135
+ //
1136
+ // Why this matters (RCA from Sentry crashes 2026-05-09
1137
+ // 21:59-22:03, all 3 .ips traces):
1138
+ // EXC_BAD_ACCESS at objc_retain+16, frame 1 = closure #1
1139
+ // in finalize+2648, queue = com.retailens.incremental.
1140
+ // stitcher. +2648 lands inside the os_log call that
1141
+ // bridges self.batchWarperType → NSString via
1142
+ // swift_bridgeObjectRetain → objc_retain. The retain
1143
+ // loaded a torn buffer pointer because:
1144
+ //
1145
+ // T0: finalize releases stateLock, dispatches workQueue
1146
+ // async closure (long stitch ahead).
1147
+ // T1: User sees fix2's "9002 No active capture" popup
1148
+ // (race between shutter-release + auto-finalize
1149
+ // useEffect, fixed in fix3 but pre-existing in
1150
+ // fix2).
1151
+ // T2: User dismisses popup, starts a new capture.
1152
+ // T3: start() acquires stateLock, sees isRunning==false,
1153
+ // WRITES self.batchWarperType = newValue. ARC
1154
+ // releases the old String buffer.
1155
+ // T4: workQueue closure (still mid-finalize from
1156
+ // capture N) loads self.batchWarperType for os_log.
1157
+ // Read interleaved with T3's write → torn String
1158
+ // buffer pointer → swift_bridgeObjectRetain →
1159
+ // objc_retain on freed memory → KERN_INVALID_ADDRESS.
1160
+ //
1161
+ // Fix3's JS dedupe closes the practical trigger (no popup,
1162
+ // no operator-confusion-driven new capture). Fix4 closes
1163
+ // the underlying race regardless of UI flow correctness.
1164
+ let batchWarperType = self.batchWarperType
1165
+ let batchBlenderType = self.batchBlenderType
1166
+ let batchSeamFinderType = self.batchSeamFinderType
1167
+ let batchEnableInscribedRectCrop = self.batchEnableInscribedRectCrop
1168
+ let keyframeExifOrientation = self.keyframeExifOrientation
1169
+ // AR-STITCHING-TWO-MODES — see memory/ar-stitching-two-modes.md
1170
+ // Snapshot the capture-time hold orientation for the bake-
1171
+ // rotation pass. Reasons same as the other snapshots above:
1172
+ // stateLock-protected against a concurrent start() rewriting
1173
+ // self.captureOrientation while the workQueue closure is mid-
1174
+ // flight.
1175
+ let captureOrientation = self.captureOrientation
1176
+ // V16 Phase 1.fix4 — pose array still held on self ivar (and
1177
+ // queued for persistent sidecar storage in a future debug-menu
1178
+ // feature) but NOT passed to the stitcher in this drop. fix4
1179
+ // uses feature-matched stitchFramePaths which doesn't take
1180
+ // poses; cv::Stitcher's BA derives camera placement from
1181
+ // features. Drop the closure-capture to avoid a compile
1182
+ // warning; ARKit pose data is preserved on the ivar regardless.
1183
+ _ = self.keyframePoses
1184
+ self.hybridEngine = nil
1185
+ self.firstwinsEngine = nil
1186
+ self.batchKeyframeMode = false
1187
+ self.keyframeCollector = nil
1188
+ self.keyframePaths = []
1189
+ self.keyframePoses = []
1190
+ self.isRunning = false
1191
+ let drops = self.droppedBackpressure
1192
+ stateLock.unlock()
1193
+
1194
+ // V16 Phase 1b.fix8 (C2 — stateless finalize):
1195
+ // The prior 7 fix attempts all individually snapshotted the
1196
+ // batch* ivars + paths/collector/engines/drops above into
1197
+ // closure-locals. This worked at the source level but the
1198
+ // `objc_retain` family of crashes kept moving from one
1199
+ // closure to the next (closure #1 → closure #2, offset
1200
+ // +2648 → +5176) — the compiler still had implicit-capture
1201
+ // latitude. C2 closes the entire class:
1202
+ // 1. Bundle every snapshot into a value-typed
1203
+ // FinalizePayload (defined at file scope above the
1204
+ // class declaration).
1205
+ // 2. Pass the payload BY VALUE into the workQueue closure
1206
+ // via an explicit capture list `[payload, completion]`.
1207
+ // 3. Bracket the closure with `C2-INVARIANT` marker
1208
+ // comments and enforce "no `self.*` inside" via the
1209
+ // CI test at scripts/check_c2_invariant.sh.
1210
+ // Net effect: the closure literally cannot read any
1211
+ // stitcher ivar. Any future edit that re-introduces a
1212
+ // `self.` reference inside the closure is caught at CI.
1213
+ let arWasRunning = inBatchKeyframeMode
1214
+ && RNSARSession.shared.isRunning
1215
+ let cleaned = (outputPath.hasPrefix("file://"))
1216
+ ? String(outputPath.dropFirst(7))
1217
+ : outputPath
1218
+ let q = max(1, min(100, jpegQuality))
1219
+ let payload = FinalizePayload(
1220
+ cleaned: cleaned,
1221
+ q: q,
1222
+ inBatchKeyframeMode: inBatchKeyframeMode,
1223
+ hybrid: hybrid,
1224
+ slit: slit,
1225
+ collector: collector,
1226
+ paths: paths,
1227
+ batchWarperType: batchWarperType,
1228
+ batchBlenderType: batchBlenderType,
1229
+ batchSeamFinderType: batchSeamFinderType,
1230
+ batchEnableInscribedRectCrop: batchEnableInscribedRectCrop,
1231
+ keyframeExifOrientation: keyframeExifOrientation,
1232
+ captureOrientation: captureOrientation,
1233
+ drops: drops,
1234
+ arWasRunning: arWasRunning
1235
+ )
1236
+
1237
+ // Then detach the AR consumer. Any in-flight delegate that
1238
+ // already captured the consumer reference will reach
1239
+ // consumeFrame, see isRunning=false, and bail.
1240
+ RNSARSession.shared.incrementalConsumer = nil
1241
+
1242
+ // V16 Phase 1b.fix1 — pause the AR session for the duration
1243
+ // of the stitch (batch-keyframe path only). ARSession holds
1244
+ // a pixel-buffer pool, world map, and plane geometry that
1245
+ // collectively contribute ~200-300 MB to baseline. Dropping
1246
+ // them while cv::Stitcher's BA + GraphCut + MultiBand runs
1247
+ // gives the stitcher more headroom under the per-process
1248
+ // limit. Restart on the main thread after the stitch
1249
+ // completes so the next capture has AR ready (next plane
1250
+ // detection + tracking re-initialise will take 2-3 s, which
1251
+ // matches Ram's chosen "Option C" trade-off).
1252
+ //
1253
+ // `arWasRunning` was computed above into FinalizePayload — read
1254
+ // from `payload.arWasRunning` here so we have one source of
1255
+ // truth (the value the closure's defer also reads).
1256
+ if payload.arWasRunning {
1257
+ os_log(.fault, log: Self.diagLog,
1258
+ "[V16-batch-keyframe] pausing AR session for stitch (memory drop)")
1259
+ RNSARSession.shared.stop()
1260
+ }
1261
+
1262
+ // V16 Phase 1b.fix6 — ARCHITECTURAL: workQueue.sync (not async)
1263
+ // for finalize.
1264
+ //
1265
+ // Background: finalize ran 5 prior fixes targeting an
1266
+ // EXC_BAD_ACCESS in objc_retain inside this closure. fix4
1267
+ // snapshotted every self.batch*, self.captureOrientation,
1268
+ // self.keyframePoses, and the engine refs into closure-locals
1269
+ // under stateLock, closing the visible torn-pointer race.
1270
+ // Three Sentry traces post-fix4 still showed the same crash
1271
+ // signature (frame 1 = closure #1 in finalize+N, queue =
1272
+ // com.retailens.incremental.stitcher), which per the
1273
+ // systematic-debugging skill (3+ fixes failed on the same
1274
+ // symptom = wrong architecture) means the workQueue.async
1275
+ // pattern itself is the problem, not any specific captured
1276
+ // ivar.
1277
+ //
1278
+ // Why .sync fixes the entire class of issues:
1279
+ // 1. Serializes finalize with start(): the bridge thread
1280
+ // can't return to JS until the stitch + completion fire.
1281
+ // JS can't trigger a new start() during the in-flight
1282
+ // stitch because the JS side awaits the finalize promise.
1283
+ // 2. Eliminates the "1-frame crash" race: with .async, when
1284
+ // paths.count==1 the closure rejected via completion +
1285
+ // returned synchronously, but the auto-finalize useEffect
1286
+ // on JS occasionally fired a SECOND finalize before the
1287
+ // first one's completion had landed. With .sync, the
1288
+ // second finalize is blocked until the first one returns.
1289
+ // 3. Removes the "what if completion's captured resolver/
1290
+ // rejecter get released by the bridge before the closure
1291
+ // runs" failure mode entirely — there's no longer any
1292
+ // window between dispatch and execution.
1293
+ //
1294
+ // Cost: bridge thread blocks for the stitch duration (2-5 s
1295
+ // on 4-6 keyframes at iPhone 16 Pro). The bridge thread is
1296
+ // NOT main (RCTEventEmitter.requiresMainQueueSetup() is false
1297
+ // and we don't override methodQueue) so UI stays responsive.
1298
+ // Other IncrementalStitcher bridge calls queue up
1299
+ // for ~3 s — acceptable since the JS side is awaiting the
1300
+ // finalize promise anyway and isn't issuing other calls
1301
+ // during that interval.
1302
+ //
1303
+ // Deadlock check: workQueue is consumed by consumeFrame
1304
+ // (AR delegate, line ~1424) and finalize. If the AR delegate
1305
+ // is currently mid-frame on workQueue when we call .sync, the
1306
+ // .sync waits for it (~50-100 ms JPEG encode) — not a
1307
+ // deadlock, just brief serialization. No other queue
1308
+ // dispatches synchronously TO workQueue, so .sync is safe.
1309
+ // MARK: C2-INVARIANT-START — no `self.` access below this line until C2-INVARIANT-END
1310
+ //
1311
+ // Enforced by scripts/check_c2_invariant.sh: any `self.` token
1312
+ // (non-comment) inside this region is a CI failure. Every
1313
+ // value the closure needs is plumbed through `payload`.
1314
+ //
1315
+ // Capture list is EXPLICIT — `[payload, completion]` ONLY.
1316
+ // The compiler will refuse any reference here that isn't
1317
+ // satisfied by these two captures, value-typed members of
1318
+ // `payload`, static types (`Self.diagLog`, `OpenCVStitcher`,
1319
+ // `FileManager`, `CGDataProvider`, `CGImage`, `NSError`,
1320
+ // `RNSARSession`), or local lets/declarations made
1321
+ // inside the closure.
1322
+ workQueue.sync { [payload, completion] in
1323
+ // V16 Phase 1b.fix1 — defer-restart AR session. Fires
1324
+ // on every exit path (success, error, early return).
1325
+ // Restart is dispatched to main because ARSession.run()
1326
+ // expects main-thread invocation to set up its rendering
1327
+ // hooks; happens AFTER the stitch completes so it doesn't
1328
+ // contend for the memory budget.
1329
+ defer {
1330
+ if payload.arWasRunning {
1331
+ // Inner closure body references only `Self.diagLog`
1332
+ // (static type) and `RNSARSession.shared`
1333
+ // (singleton) — both name-resolve without
1334
+ // capturing self. No explicit capture list
1335
+ // required; the C2 invariant script grep-checks
1336
+ // for `self.*` tokens which this body has none of.
1337
+ DispatchQueue.main.async {
1338
+ os_log(.fault, log: Self.diagLog,
1339
+ "[V16-batch-keyframe] restarting AR session post-stitch")
1340
+ RNSARSession.shared.start()
1341
+ }
1342
+ }
1343
+ }
1344
+ do {
1345
+ if payload.inBatchKeyframeMode {
1346
+ // V16 Phase 1 — hand collected keyframes + poses
1347
+ // to OpenCVStitcher's full BA + GraphCut +
1348
+ // ExposureComp + MultiBand pipeline. ≤6 frames
1349
+ // means BA stays bounded and MultiBand fits.
1350
+ os_log(.fault, log: Self.diagLog,
1351
+ "[V16-batch-keyframe] finalize ENTRY frames=%d",
1352
+ payload.paths.count)
1353
+ // V16 Phase 1b.fix7 — single-keyframe UX:
1354
+ // accept a 1-frame finalize by copying the lone
1355
+ // keyframe JPEG to outputPath (preserving the JPEG
1356
+ // bytes as-is, no re-encode). Previously this path
1357
+ // rejected with code 9003, and an auto-finalize
1358
+ // useEffect on JS could fire it during a quick
1359
+ // tap-and-release. The user-visible failure
1360
+ // ("9003 only 1 keyframe saved") was both
1361
+ // confusing AND occasionally racing with a second
1362
+ // finalize call into the EXC_BAD_ACCESS path that
1363
+ // fix4/fix6 closed. Returning the single frame as
1364
+ // the output is the right UX: a single keyframe IS
1365
+ // a valid panorama capture (just one shot).
1366
+ if payload.paths.count < 2 {
1367
+ if payload.paths.count == 1 {
1368
+ let src = payload.paths[0]
1369
+ do {
1370
+ // Remove any pre-existing file at the
1371
+ // output path — copyItem refuses to
1372
+ // overwrite, and a stale tmp file from
1373
+ // a prior auto-finalize attempt is the
1374
+ // common case.
1375
+ let fm = FileManager.default
1376
+ if fm.fileExists(atPath: payload.cleaned) {
1377
+ try fm.removeItem(atPath: payload.cleaned)
1378
+ }
1379
+ try fm.copyItem(atPath: src, toPath: payload.cleaned)
1380
+ // Read back the JPEG dimensions for
1381
+ // the result dictionary — match
1382
+ // OpenCVStitcher.stitchFramePaths'
1383
+ // {width, height} contract so JS
1384
+ // doesn't have to special-case a
1385
+ // single-frame result.
1386
+ var width: Int = 0
1387
+ var height: Int = 0
1388
+ if let provider = CGDataProvider(filename: payload.cleaned),
1389
+ let img = CGImage(
1390
+ jpegDataProviderSource: provider,
1391
+ decode: nil,
1392
+ shouldInterpolate: false,
1393
+ intent: .defaultIntent) {
1394
+ width = img.width
1395
+ height = img.height
1396
+ }
1397
+ os_log(.fault, log: Self.diagLog,
1398
+ "[V16-batch-keyframe.fix7] single-keyframe finalize: copied %{public}@ → %{public}@ (%dx%d)",
1399
+ src, payload.cleaned,
1400
+ Int32(width), Int32(height))
1401
+ completion([
1402
+ "panoramaPath": payload.cleaned,
1403
+ "width": width,
1404
+ "height": height,
1405
+ "acceptedCount": 1,
1406
+ "droppedBackpressure": payload.drops,
1407
+ "batchKeyframeSessionDir":
1408
+ payload.collector?.sessionDir ?? "",
1409
+ "batchKeyframeCount": 1,
1410
+ "singleKeyframe": true,
1411
+ ], nil)
1412
+ return
1413
+ } catch let copyErr as NSError {
1414
+ // Fall through to the legacy "not
1415
+ // enough keyframes" rejection so the
1416
+ // user at least gets a discoverable
1417
+ // error rather than a silent hang.
1418
+ os_log(.fault, log: Self.diagLog,
1419
+ "[V16-batch-keyframe.fix7] single-keyframe copy failed: %{public}@",
1420
+ copyErr.localizedDescription)
1421
+ }
1422
+ }
1423
+ payload.collector?.cleanup()
1424
+ completion(nil, NSError(
1425
+ domain: "RNImageStitcherIncremental",
1426
+ code: 9003,
1427
+ userInfo: [NSLocalizedDescriptionKey:
1428
+ "Batch-keyframe finalize: 0 keyframes saved — capture didn't accept any frames."]
1429
+ ))
1430
+ return
1431
+ }
1432
+ // Swift bridges `(NSError**)error` as `throws`,
1433
+ // so we use a do/catch instead of an inout error
1434
+ // pointer. Result is non-optional inside the
1435
+ // try block (or we wouldn't reach the success
1436
+ // branch).
1437
+ do {
1438
+ // V16 Phase 1.fix4 — ARCHITECTURAL PIVOT.
1439
+ // Three iterations of pose-driven (fix1/2/3)
1440
+ // produced different failure modes (UMat
1441
+ // bbox blowup, sideways output, frames out of
1442
+ // order, same physical object placed twice).
1443
+ // Per the systematic-debugging skill: 3+ failed
1444
+ // fixes on the same approach = wrong
1445
+ // architecture.
1446
+ //
1447
+ // Switching to cv::Stitcher's feature-matched
1448
+ // pipeline (stitchFramePaths) — same warper /
1449
+ // blender / seam settings, but uses ORB +
1450
+ // BFMatcher + RANSAC + BundleAdjusterRay +
1451
+ // waveCorrect for camera placement instead of
1452
+ // ARKit poses. Battle-tested for years in
1453
+ // cv::Stitcher::PANORAMA / SCANS modes.
1454
+ //
1455
+ // The 4-6 keyframes (with ≥40% new content
1456
+ // each, guaranteed by the Phase 0 gate) have
1457
+ // 60% overlap on retail content — features
1458
+ // are abundant and BA converges in <500 ms.
1459
+ // Output orientation, frame ordering, and
1460
+ // canvas bounds are all determined BY THE
1461
+ // FEATURES, not by pose-convention assumptions.
1462
+ //
1463
+ // ARKit poses are still saved alongside each
1464
+ // keyframe (`keyframePoses`) for future
1465
+ // pose-driven investigation as a separate
1466
+ // workstream — that path stays in the codebase
1467
+ // (stitchKeyframePaths method) but isn't on
1468
+ // the hot path.
1469
+ // V16 Phase 1b.fix3 — pass the EXIF Orientation
1470
+ // tag derived from `frameRotationDegrees`.
1471
+ // V16 Phase 1b.fix8 (C2) — read knobs from
1472
+ // `payload` (value-snapshot built under
1473
+ // stateLock in finalize's prologue). No
1474
+ // `self.*` access here; closure cannot race
1475
+ // with a concurrent `start()`.
1476
+ // AR-STITCHING-TWO-MODES — see memory/ar-stitching-two-modes.md
1477
+ // `payload.captureOrientation` drives the .mm
1478
+ // bake-rotation; `payload.keyframeExifOrientation`
1479
+ // kept for any future per-keyframe EXIF needs.
1480
+ _ = payload.keyframeExifOrientation
1481
+ os_log(.fault, log: Self.diagLog,
1482
+ "[V16-batch-keyframe] stitch (feature-matched): warper=%{public}@ blender=%{public}@ seam=%{public}@ captureOrientation=%{public}@",
1483
+ payload.batchWarperType,
1484
+ payload.batchBlenderType,
1485
+ payload.batchSeamFinderType,
1486
+ payload.captureOrientation)
1487
+ let r = try OpenCVStitcher.stitchFramePaths(
1488
+ payload.paths,
1489
+ outputPath: payload.cleaned,
1490
+ jpegQuality: payload.q,
1491
+ warperType: payload.batchWarperType,
1492
+ blenderType: payload.batchBlenderType,
1493
+ seamFinderType: payload.batchSeamFinderType,
1494
+ captureOrientation: payload.captureOrientation,
1495
+ useInscribedRectCrop: payload.batchEnableInscribedRectCrop
1496
+ )
1497
+ // V16 fix-attempt 9 (verified on device,
1498
+ // 2026-05-13) — sentinel-result detection.
1499
+ //
1500
+ // Background: 8 prior fix attempts (fix1-fix8)
1501
+ // chased a deterministic SEGV in Swift's
1502
+ // try-bridge over OpenCVStitcher's NSError-
1503
+ // out-parameter return. ASan with Sentry
1504
+ // disabled (.ips 172125) localised the SEGV to
1505
+ // an objc_retain on a wild pointer in unmapped
1506
+ // VM (0x60007a530, ReportDeadlySignal — no
1507
+ // shadow-memory match) immediately after the
1508
+ // .mm `return nil`. The fix-9 NULL TEST
1509
+ // (2026-05-13 17:21) replaced the two
1510
+ // immediate-repro `*error+return nil` sites
1511
+ // with non-nil sentinel returns; the crash
1512
+ // went away cleanly. After the test passed
1513
+ // we extended the sentinel pattern to ALL six
1514
+ // failure-return sites in stitchFramePaths
1515
+ // (pre-stitch memory abort, frames<2,
1516
+ // loadFramesOrFail, validPairs<1, workFrames<2,
1517
+ // estimator failure) so a production trigger of
1518
+ // any of them produces a clean error surface
1519
+ // instead of crashing the same way.
1520
+ //
1521
+ // We can't tell which underlying cause produced
1522
+ // a given sentinel from Swift (the .mm logs the
1523
+ // specific reason via NSLog so Console.app
1524
+ // shows it). JS gets one generic error code
1525
+ // (9007); refining the JS-facing taxonomy is
1526
+ // a follow-up if/when product needs differ-
1527
+ // entiated UX per cause.
1528
+ //
1529
+ // See: docs/site-content/design/2026-05-12-finalize-crash-investigation.md
1530
+ if r.width == 0 && r.height == 0 {
1531
+ os_log(.fault, log: Self.diagLog,
1532
+ "[V16-batch-keyframe.fix9] sentinel result from stitchFramePaths — see preceding [BatchStitcher] NSLog for cause; emitting clean error to JS")
1533
+ completion(nil, NSError(
1534
+ domain: "RNImageStitcherIncremental",
1535
+ code: 9007,
1536
+ userInfo: [NSLocalizedDescriptionKey:
1537
+ "Could not stitch the captured frames into a panorama. Please try recapturing with a slower, more deliberate pan that overlaps each section by at least 50%."]
1538
+ ))
1539
+ return
1540
+ }
1541
+ // Keep saved keyframes on disk for post-hoc
1542
+ // re-processing (Ram's request). Cleanup is
1543
+ // a follow-up debug-menu task.
1544
+ // 2026-05-16 (Issue 5) — surface C+D
1545
+ // progressive-confidence retry telemetry to JS
1546
+ // so the host can render a debug toast. -1
1547
+ // sentinels = "no retry data" (early-return
1548
+ // success paths bypass the retry loop).
1549
+ var batchDict: [String: Any] = [
1550
+ "panoramaPath": r.outputPath,
1551
+ "width": Int(r.width),
1552
+ "height": Int(r.height),
1553
+ "acceptedCount": payload.paths.count,
1554
+ "droppedBackpressure": payload.drops,
1555
+ "batchKeyframeSessionDir":
1556
+ payload.collector?.sessionDir ?? "",
1557
+ "batchKeyframeCount": payload.paths.count,
1558
+ ]
1559
+ if r.framesRequested >= 0 {
1560
+ batchDict["framesRequested"] = Int(r.framesRequested)
1561
+ }
1562
+ if r.framesIncluded >= 0 {
1563
+ batchDict["framesIncluded"] = Int(r.framesIncluded)
1564
+ if r.framesRequested >= 0 {
1565
+ batchDict["framesDropped"] =
1566
+ Int(r.framesRequested - r.framesIncluded)
1567
+ }
1568
+ }
1569
+ if r.finalConfidenceThresh >= 0 {
1570
+ batchDict["finalConfidenceThresh"] = r.finalConfidenceThresh
1571
+ }
1572
+ completion(batchDict, nil)
1573
+ } catch let stitchErr as NSError {
1574
+ completion(nil, stitchErr)
1575
+ }
1576
+ } else if let hybrid = payload.hybrid {
1577
+ let snap = try hybrid.finalize(atPath: payload.cleaned, jpegQuality: payload.q)
1578
+ completion([
1579
+ "panoramaPath": snap.panoramaPath,
1580
+ "width": snap.width,
1581
+ "height": snap.height,
1582
+ "acceptedCount": snap.acceptedCount,
1583
+ "droppedBackpressure": payload.drops,
1584
+ ], nil)
1585
+ // 2026-05-16 — realtime+batch fusion (Option A
1586
+ // "Replace on completion") hook. The live
1587
+ // panorama has been written and the JS finalize
1588
+ // promise has resolved; now fire-and-forget an
1589
+ // async refinement over the hybrid engine's
1590
+ // accepted keyframes.
1591
+ //
1592
+ // Constraints honoured here (per the design doc
1593
+ // and the prompt's "Constraints" list):
1594
+ // 1. Hybrid realtime engine is NOT modified —
1595
+ // `OpenCVIncrementalStitcher.mm` stays
1596
+ // untouched; we only consult the existing
1597
+ // keyframe-path ivar that finalize() already
1598
+ // snapshotted into `payload.paths`.
1599
+ // 2. NO-OP when keyframes are not on disk.
1600
+ // Today's hybrid engine does NOT save per-
1601
+ // frame JPEGs (only batch-keyframe mode does
1602
+ // via OpenCVKeyframeCollector), so
1603
+ // `payload.paths` is empty for the hybrid
1604
+ // branch. `runHybridAutoRefine` detects
1605
+ // that and emits `isRefining=false` without
1606
+ // running cv::Stitcher. When a future change
1607
+ // hooks the hybrid engine up to a keyframe
1608
+ // collector, the same code path lights up
1609
+ // automatically.
1610
+ // 3. Refinement is fire-and-forget — finalize's
1611
+ // promise has ALREADY been resolved above.
1612
+ //
1613
+ // Capture-list discipline (C2 invariant — see the
1614
+ // file-top markers). No `self.*` references allowed
1615
+ // here; we route the dispatch through the type
1616
+ // (IncrementalStitcher.shared) so the
1617
+ // closure captures only value-typed locals + the
1618
+ // class type itself. shared is a process-wide
1619
+ // singleton (initialised once at module load),
1620
+ // so this is lifecycle-safe.
1621
+ let refinedOut = Self.refinedPathFromLive(
1622
+ livePath: snap.panoramaPath
1623
+ )
1624
+ let pathsForRefine = payload.paths // empty for hybrid today
1625
+ let capOri = payload.captureOrientation
1626
+ let warper = payload.batchWarperType
1627
+ let blender = payload.batchBlenderType
1628
+ let seam = payload.batchSeamFinderType
1629
+ let inscribed = payload.batchEnableInscribedRectCrop
1630
+ IncrementalStitcher.shared.refineQueue.async {
1631
+ IncrementalStitcher.shared.runHybridAutoRefine(
1632
+ framePaths: pathsForRefine,
1633
+ refinedOutputPath: refinedOut,
1634
+ captureOrientation: capOri,
1635
+ warperType: warper,
1636
+ blenderType: blender,
1637
+ seamFinderType: seam,
1638
+ useInscribedRectCrop: inscribed
1639
+ )
1640
+ }
1641
+ } else if let slit = payload.slit {
1642
+ let snap = try slit.finalize(atPath: payload.cleaned, jpegQuality: payload.q)
1643
+ completion([
1644
+ "panoramaPath": snap.panoramaPath,
1645
+ "width": snap.width,
1646
+ "height": snap.height,
1647
+ "acceptedCount": snap.acceptedCount,
1648
+ "droppedBackpressure": payload.drops,
1649
+ ], nil)
1650
+ } else {
1651
+ completion(nil, NSError(
1652
+ domain: "RNImageStitcherIncremental",
1653
+ code: 9002,
1654
+ userInfo: [NSLocalizedDescriptionKey:
1655
+ "No active capture — call start() first."]
1656
+ ))
1657
+ }
1658
+ } catch let err as NSError {
1659
+ completion(nil, err)
1660
+ }
1661
+ }
1662
+ // MARK: C2-INVARIANT-END
1663
+ }
1664
+
1665
+ /// 2026-05-16 — realtime+batch fusion (Option A) entry point.
1666
+ /// Runs the shared C++ stitcher over the supplied keyframe JPEGs
1667
+ /// and writes a refined panorama to `outputPath`.
1668
+ ///
1669
+ /// Called by:
1670
+ /// 1. The bridge layer (explicit JS `refinePanorama(...)` API).
1671
+ /// 2. `runHybridAutoRefine(...)` below, the fire-and-forget hook
1672
+ /// from `finalize()` for the hybrid-engine path.
1673
+ ///
1674
+ /// Threading: the work itself dispatches onto `refineQueue` (NOT
1675
+ /// `workQueue`). That keeps the per-capture path completely
1676
+ /// independent — a refinement in flight does not delay a fresh
1677
+ /// start()/consumeFrame() pair the operator may have initiated
1678
+ /// while the refinement runs. Completion fires on `refineQueue`;
1679
+ /// callers that need main-thread delivery (e.g. the bridge
1680
+ /// promise resolver) re-dispatch as needed.
1681
+ ///
1682
+ /// Configuration: same option set the bridge sees from JS plus
1683
+ /// production-tested defaults that match the existing
1684
+ /// batch-keyframe finalize path:
1685
+ /// warperType = "spherical"
1686
+ /// blenderType = "multiband"
1687
+ /// seamFinderType = "graphcut"
1688
+ /// captureOrientation = "portrait"
1689
+ /// useInscribedRectCrop = false
1690
+ /// jpegQuality = 90
1691
+ ///
1692
+ /// Pre-conditions enforced here (in addition to bridge-level
1693
+ /// validation): every input path must exist on disk; if any is
1694
+ /// missing the call resolves with an NSError so the caller can
1695
+ /// surface a clean error rather than letting cv::imread crash
1696
+ /// inside the manual pipeline.
1697
+ @objc public func refinePanorama(
1698
+ framePaths: [String],
1699
+ outputPath: String,
1700
+ config: [String: Any],
1701
+ completion: @escaping ([String: Any]?, NSError?) -> Void
1702
+ ) {
1703
+ guard framePaths.count >= 2 else {
1704
+ completion(nil, NSError(
1705
+ domain: "RNImageStitcherIncremental",
1706
+ code: 9101,
1707
+ userInfo: [NSLocalizedDescriptionKey:
1708
+ "refinePanorama requires at least 2 framePaths (got \(framePaths.count))."]
1709
+ ))
1710
+ return
1711
+ }
1712
+ let fm = FileManager.default
1713
+ for p in framePaths {
1714
+ let cleaned = p.hasPrefix("file://") ? String(p.dropFirst(7)) : p
1715
+ if !fm.fileExists(atPath: cleaned) {
1716
+ completion(nil, NSError(
1717
+ domain: "RNImageStitcherIncremental",
1718
+ code: 9102,
1719
+ userInfo: [NSLocalizedDescriptionKey:
1720
+ "refinePanorama: keyframe missing on disk — \(cleaned)"]
1721
+ ))
1722
+ return
1723
+ }
1724
+ }
1725
+ let warper = (config["warperType"] as? String) ?? "spherical"
1726
+ let blender = (config["blenderType"] as? String) ?? "multiband"
1727
+ let seam = (config["seamFinderType"] as? String) ?? "graphcut"
1728
+ let orientation = (config["captureOrientation"] as? String) ?? "portrait"
1729
+ let useInscribed = (config["useInscribedRectCrop"] as? Bool) ?? false
1730
+ let quality = max(1, min(100, (config["jpegQuality"] as? Int) ?? 90))
1731
+ let cleanedOutput = outputPath.hasPrefix("file://")
1732
+ ? String(outputPath.dropFirst(7))
1733
+ : outputPath
1734
+
1735
+ os_log(.fault, log: Self.diagLog,
1736
+ "[refine] dispatch frames=%d output=%{public}@ warper=%{public}@ blender=%{public}@ seam=%{public}@",
1737
+ framePaths.count,
1738
+ cleanedOutput,
1739
+ warper, blender, seam)
1740
+
1741
+ refineQueue.async {
1742
+ // C2-style: closure captures only value-typed locals
1743
+ // (paths, output path, config strings). No `self` access
1744
+ // is needed for the cv::Stitcher call — OpenCVStitcher is
1745
+ // a class method, not an instance method, so we can call
1746
+ // it directly via the type.
1747
+ do {
1748
+ let r = try OpenCVStitcher.stitchFramePaths(
1749
+ framePaths,
1750
+ outputPath: cleanedOutput,
1751
+ jpegQuality: quality,
1752
+ warperType: warper,
1753
+ blenderType: blender,
1754
+ seamFinderType: seam,
1755
+ captureOrientation: orientation,
1756
+ useInscribedRectCrop: useInscribed
1757
+ )
1758
+ // fix-9 sentinel detection — see the finalize() path
1759
+ // for the full rationale. A 0×0 result means
1760
+ // OpenCVStitcher hit one of its six guarded failure
1761
+ // returns; surface as a clean NSError.
1762
+ if r.width == 0 && r.height == 0 {
1763
+ completion(nil, NSError(
1764
+ domain: "RNImageStitcherIncremental",
1765
+ code: 9107,
1766
+ userInfo: [NSLocalizedDescriptionKey:
1767
+ "refinePanorama: stitcher returned sentinel — see preceding [BatchStitcher] log for cause."]
1768
+ ))
1769
+ return
1770
+ }
1771
+ completion([
1772
+ "panoramaPath": r.outputPath,
1773
+ "width": Int(r.width),
1774
+ "height": Int(r.height),
1775
+ "framesRequested": framePaths.count,
1776
+ "framesIncluded": framePaths.count,
1777
+ "framesDropped": 0,
1778
+ "finalConfidenceThresh": -1.0,
1779
+ ], nil)
1780
+ } catch let err as NSError {
1781
+ completion(nil, err)
1782
+ }
1783
+ }
1784
+ }
1785
+
1786
+ /// 2026-05-16 — realtime+batch fusion (Option A) auto-trigger.
1787
+ /// Called from `finalize()` immediately after the hybrid engine
1788
+ /// wrote its live panorama; fire-and-forget from finalize()'s
1789
+ /// perspective so the JS-side finalize promise resolves with the
1790
+ /// live result first. Then this method:
1791
+ ///
1792
+ /// 1. Emits a state event with `isRefining = true` so the host
1793
+ /// can render its "Refining…" pill.
1794
+ /// 2. Runs `refinePanorama(framePaths, refinedOutputPath, ...)`.
1795
+ /// 3. On success: emits a state event with `isRefining = false`
1796
+ /// AND `refinedPanoramaPath = <path>` so the host swaps in
1797
+ /// the higher-quality output.
1798
+ /// 4. On failure: emits a state event with `isRefining = false`
1799
+ /// AND NO refined path. Host keeps showing the live
1800
+ /// panorama; the design doc's "Couldn't refine" toast UX is
1801
+ /// a follow-up.
1802
+ ///
1803
+ /// No-op when `framePaths.count < 2` or any framePath is missing
1804
+ /// on disk. Hybrid-engine captures DO NOT today save per-frame
1805
+ /// JPEGs, so this method's most common call site (from finalize's
1806
+ /// hybrid branch) currently produces a no-op + isRefining=false
1807
+ /// emit — which is intentional (the design doc says "if
1808
+ /// keyframes are NOT on disk, the auto-trigger is a no-op").
1809
+ private func runHybridAutoRefine(
1810
+ framePaths: [String],
1811
+ refinedOutputPath: String,
1812
+ captureOrientation: String,
1813
+ warperType: String,
1814
+ blenderType: String,
1815
+ seamFinderType: String,
1816
+ useInscribedRectCrop: Bool
1817
+ ) {
1818
+ if framePaths.count < 2 {
1819
+ os_log(.info, log: Self.diagLog,
1820
+ "[refine.auto] skipped: framePaths.count=%d (< 2 — hybrid engine retains no per-frame JPEGs)",
1821
+ framePaths.count)
1822
+ // Emit isRefining=false so any host that pre-seeded a
1823
+ // pill on finalize doesn't get stuck.
1824
+ self.emitRefinementState(isRefining: false, refinedPanoramaPath: nil)
1825
+ return
1826
+ }
1827
+ // Pre-flight existence check so we degrade gracefully when
1828
+ // a JPEG was unlinked between finalize and the dispatch
1829
+ // landing on refineQueue.
1830
+ let fm = FileManager.default
1831
+ for p in framePaths {
1832
+ let cleaned = p.hasPrefix("file://") ? String(p.dropFirst(7)) : p
1833
+ if !fm.fileExists(atPath: cleaned) {
1834
+ os_log(.info, log: Self.diagLog,
1835
+ "[refine.auto] skipped: missing keyframe %{public}@",
1836
+ cleaned)
1837
+ self.emitRefinementState(isRefining: false, refinedPanoramaPath: nil)
1838
+ return
1839
+ }
1840
+ }
1841
+ // Signal the pill on before the stitcher work begins. The
1842
+ // emit goes through the same notification channel as every
1843
+ // other state update; JS sees it asynchronously, which is
1844
+ // fine — operator UX wants the pill within a few hundred ms,
1845
+ // not synchronously with finalize's promise resolution.
1846
+ self.emitRefinementState(isRefining: true, refinedPanoramaPath: nil)
1847
+ let config: [String: Any] = [
1848
+ "warperType": warperType,
1849
+ "blenderType": blenderType,
1850
+ "seamFinderType": seamFinderType,
1851
+ "captureOrientation": captureOrientation,
1852
+ "useInscribedRectCrop": useInscribedRectCrop,
1853
+ "jpegQuality": 90,
1854
+ ]
1855
+ self.refinePanorama(
1856
+ framePaths: framePaths,
1857
+ outputPath: refinedOutputPath,
1858
+ config: config
1859
+ ) { [weak self] result, error in
1860
+ guard let self = self else { return }
1861
+ if let error = error {
1862
+ os_log(.fault, log: Self.diagLog,
1863
+ "[refine.auto] refinement failed: %{public}@ — leaving live output in place",
1864
+ error.localizedDescription)
1865
+ self.emitRefinementState(isRefining: false, refinedPanoramaPath: nil)
1866
+ return
1867
+ }
1868
+ let path = (result?["panoramaPath"] as? String) ?? refinedOutputPath
1869
+ os_log(.fault, log: Self.diagLog,
1870
+ "[refine.auto] success path=%{public}@",
1871
+ path)
1872
+ self.emitRefinementState(isRefining: false, refinedPanoramaPath: path)
1873
+ }
1874
+ }
1875
+
1876
+ /// 2026-05-16 — emit a minimal state event carrying only the
1877
+ /// refinement-related fields. Mirrors the existing
1878
+ /// `emitBatchKeyframeAcceptedState` pattern: build a fresh
1879
+ /// IncrementalStateObject, then add the new optional fields
1880
+ /// directly to the userInfo dict so JS (which reads from the
1881
+ /// raw event payload) picks them up without a schema change in
1882
+ /// the Obj-C class.
1883
+ private func emitRefinementState(
1884
+ isRefining: Bool,
1885
+ refinedPanoramaPath: String?
1886
+ ) {
1887
+ // Preserve the most-recent panoramaPath / dims / accepted
1888
+ // count so the JS subscriber's sticky-snapshot merge keeps
1889
+ // showing the live preview between the finalize and the
1890
+ // refined swap. All other fields default to "no-op" values.
1891
+ stateLock.lock()
1892
+ let prev = self.lastState
1893
+ stateLock.unlock()
1894
+ let state = IncrementalStateObject(
1895
+ panoramaPath: prev?.panoramaPath,
1896
+ width: prev?.width ?? 0,
1897
+ height: prev?.height ?? 0,
1898
+ acceptedCount: prev?.acceptedCount ?? 0,
1899
+ outcome: prev?.outcome ?? .acceptedHigh,
1900
+ confidence: prev?.confidence ?? 1.0,
1901
+ overlapPercent: prev?.overlapPercent ?? -1.0,
1902
+ processingMs: 0,
1903
+ isLandscape: prev?.isLandscape ?? false,
1904
+ paintedExtent: prev?.paintedExtent ?? 0,
1905
+ panExtent: prev?.panExtent ?? 0,
1906
+ keyframeMax: prev?.keyframeMax ?? 0
1907
+ )
1908
+ var dict = state.asDictionary()
1909
+ dict["isRefining"] = isRefining
1910
+ if let p = refinedPanoramaPath {
1911
+ dict["refinedPanoramaPath"] = p
1912
+ }
1913
+ NotificationCenter.default.post(
1914
+ name: .retailensIncrementalStateUpdate,
1915
+ object: nil,
1916
+ userInfo: dict
1917
+ )
1918
+ }
1919
+
1920
+ /// Cancel an in-progress capture without producing output.
1921
+ /// Same V12.1 synchronous-stop pattern as finalize.
1922
+ @objc public func cancel() {
1923
+ // V12.9 fix #3 — same ordering as finalize: flip isRunning
1924
+ // FIRST so any in-flight consumeFrame bails at its first
1925
+ // guard. Then detach the AR consumer.
1926
+ stateLock.lock()
1927
+ let hybrid = self.hybridEngine
1928
+ let slit = self.firstwinsEngine
1929
+ let collector = self.keyframeCollector
1930
+ self.hybridEngine = nil
1931
+ self.firstwinsEngine = nil
1932
+ self.keyframeCollector = nil
1933
+ self.batchKeyframeMode = false
1934
+ self.keyframePaths = []
1935
+ self.keyframePoses = []
1936
+ self.isRunning = false
1937
+ self.lastState = nil
1938
+ // V16 — reset the keyframe gate so the next start() begins
1939
+ // with a clean polygon state and counter. Safe to do under
1940
+ // stateLock because the gate is only mutated from the AR
1941
+ // delegate (consumeFrame) and the JS thread (start/cancel
1942
+ // /markNextFrameAsLastKeyframe), all serialized via this lock.
1943
+ self.keyframeGate.reset()
1944
+ stateLock.unlock()
1945
+ RNSARSession.shared.incrementalConsumer = nil
1946
+ // Reset on the work queue so we don't race with an in-flight
1947
+ // ingest that's still touching the engine's canvas. Cancel
1948
+ // ALSO removes the collector's session directory — the
1949
+ // operator explicitly aborted, so the saved JPEGs aren't
1950
+ // worth keeping for re-processing.
1951
+ workQueue.async {
1952
+ hybrid?.reset()
1953
+ slit?.reset()
1954
+ collector?.cleanup()
1955
+ }
1956
+ }
1957
+
1958
+ /// V16 — JS-side hook for shutter-release: arm the gate so the
1959
+ /// NEXT delivered ARFrame is force-accepted regardless of overlap.
1960
+ /// Without this, the user releasing mid-pan (between two natural
1961
+ /// keyframe boundaries) would leave the trailing edge of the
1962
+ /// scene unrepresented in the panorama.
1963
+ ///
1964
+ /// Idempotent: setting the flag while it's already set is a no-op.
1965
+ /// Safe to call from any thread (NSLock-guarded). No-op when the
1966
+ /// gate is disabled (frameSelectionMode = "time-based").
1967
+ @objc public func markNextFrameAsLastKeyframe() {
1968
+ stateLock.lock()
1969
+ defer { stateLock.unlock() }
1970
+ guard self.isRunning, self.keyframeGate.enabled else { return }
1971
+ self.keyframeGate.forceAcceptNext = true
1972
+ os_log(.fault, log: Self.diagLog,
1973
+ "[V16-keyframe] markNextFrameAsLastKeyframe armed (count=%d max=%d)",
1974
+ self.keyframeGate.acceptedCount, self.keyframeGate.maxCount)
1975
+ }
1976
+
1977
+ /// Whether the engine is currently in batch-keyframe mode.
1978
+ /// Bridge reads this to decide whether the JS-driven
1979
+ /// `processFrameAtPath` path can use the lightweight
1980
+ /// `addBatchKeyframePath` (path-only) entry below.
1981
+ @objc public var isBatchKeyframeMode: Bool {
1982
+ stateLock.lock()
1983
+ defer { stateLock.unlock() }
1984
+ return batchKeyframeMode
1985
+ }
1986
+
1987
+ /// 2026-05-18 (Issue #2 v2) — JS-driver entry-point for
1988
+ /// batch-keyframe captures. Mirrors Android's behaviour: the
1989
+ /// caller (JS side via the IncrementalStitcherBridge) hands us a
1990
+ /// JPEG file path that already exists on disk (saved by
1991
+ /// vision-camera's takeSnapshot), plus a synthetic pose derived
1992
+ /// from gyro integration. We:
1993
+ ///
1994
+ /// 1. Validate state (running + batchKeyframeMode).
1995
+ /// 2. Ask the shared C++ KeyframeGate whether to accept this
1996
+ /// frame. Pass `latchedPlane: nil` — non-AR captures have
1997
+ /// no plane; the C++ gate falls back to a pose-only
1998
+ /// angular-delta strategy. We do NOT pass a pixel buffer:
1999
+ /// the Pose strategy doesn't need one, and avoiding the
2000
+ /// JPEG → CVPixelBuffer round-trip dodges the iOS
2001
+ /// orientation bugs that broke Issue 2 v1
2002
+ /// (UIImage/CGContext Y-flip + EXIF-vs-CGImage dimension
2003
+ /// mismatch — see the symptom in 2026-05-18 user report).
2004
+ /// 3. If accepted, append the existing path + pose to the
2005
+ /// finalize-time lists. No JPEG re-encode — the file on
2006
+ /// disk IS the keyframe. `retailens::stitchFramePaths()`
2007
+ /// at finalize uses `cv::imread` which natively handles
2008
+ /// EXIF orientation, so the output panorama reads upright.
2009
+ /// 4. Emit the same state-event the AR delegate path emits so
2010
+ /// the JS live band populates identically.
2011
+ ///
2012
+ /// Architecture note: this is structurally parallel to Android's
2013
+ /// `IncrementalStitcher.kt::processFrameAtPath`
2014
+ /// `batchKeyframeMode` branch (lines 573-627). A follow-up
2015
+ /// should extract the dispatch (gate-eval + path-append + emit)
2016
+ /// into shared cpp/ so both platforms become 5-line wrappers
2017
+ /// around a single C++ entry point.
2018
+ @objc public func addBatchKeyframePath(
2019
+ path: String,
2020
+ pose: RNSARFramePose
2021
+ ) -> Bool {
2022
+ stateLock.lock()
2023
+ guard self.isRunning, self.batchKeyframeMode else {
2024
+ stateLock.unlock()
2025
+ return false
2026
+ }
2027
+ stateLock.unlock()
2028
+
2029
+ // Pose-only gate evaluation — no pixel buffer, no plane.
2030
+ let decision = self.keyframeGate.evaluate(
2031
+ pose: pose,
2032
+ latchedPlane: nil
2033
+ )
2034
+ if !decision.accept {
2035
+ self.emitKeyframeRejectState(decision: decision)
2036
+ return false
2037
+ }
2038
+
2039
+ // Append path + pose to the finalize lists. Take the lock
2040
+ // briefly — these mutate state read by `finalize()`.
2041
+ stateLock.lock()
2042
+ self.keyframePaths.append(path)
2043
+ self.keyframePoses.append(pose.asDictionary())
2044
+ let count = self.keyframePaths.count
2045
+ stateLock.unlock()
2046
+ os_log(.fault, log: Self.diagLog,
2047
+ "[V16-batch-keyframe.js] accepted path #%d → %{public}@",
2048
+ Int32(count), path)
2049
+ self.emitBatchKeyframeAcceptedState(
2050
+ thumbnailPath: path,
2051
+ keyframeIndex: count - 1,
2052
+ keyframeCount: count,
2053
+ keyframeMax: self.keyframeGate.maxCount,
2054
+ isLandscape: pose.imageWidth >= pose.imageHeight
2055
+ )
2056
+ return true
2057
+ }
2058
+
2059
+ /// V16 Phase 1 — emit a state event when a batch-keyframe is
2060
+ /// saved. Carries the on-disk thumbnail path so JS can render it
2061
+ /// in LiveFrameStrip + advance the "Keyframes: N/M" pill.
2062
+ private func emitBatchKeyframeAcceptedState(
2063
+ thumbnailPath: String,
2064
+ keyframeIndex: Int,
2065
+ keyframeCount: Int,
2066
+ keyframeMax: Int,
2067
+ isLandscape: Bool
2068
+ ) {
2069
+ let state = IncrementalStateObject(
2070
+ panoramaPath: nil,
2071
+ width: 0,
2072
+ height: 0,
2073
+ acceptedCount: keyframeCount,
2074
+ outcome: .acceptedHigh,
2075
+ confidence: 1.0,
2076
+ overlapPercent: -1.0,
2077
+ processingMs: 0,
2078
+ isLandscape: isLandscape,
2079
+ paintedExtent: 0,
2080
+ panExtent: 0,
2081
+ keyframeMax: keyframeMax
2082
+ )
2083
+ stateLock.lock()
2084
+ self.lastState = state
2085
+ stateLock.unlock()
2086
+ var dict = state.asDictionary()
2087
+ // Extra fields the existing IncrementalState schema doesn't
2088
+ // carry — JS reads these directly from the userInfo blob.
2089
+ dict["batchKeyframeThumbnailPath"] = thumbnailPath
2090
+ dict["batchKeyframeIndex"] = keyframeIndex
2091
+ NotificationCenter.default.post(
2092
+ name: .retailensIncrementalStateUpdate,
2093
+ object: nil,
2094
+ userInfo: dict
2095
+ )
2096
+ }
2097
+
2098
+ /// V16 Phase 1b.fix2 — deep-copy a CVPixelBuffer so it survives
2099
+ /// past ARKit's pool reuse window. Apple's contract is that
2100
+ /// ARFrame.capturedImage is only valid inside the delegate
2101
+ /// scope; CFRetain alone doesn't extend the underlying
2102
+ /// IOSurface's lifetime. This is the documented fix.
2103
+ ///
2104
+ /// Handles both planar (NV12 — ARKit default) and packed
2105
+ /// (BGRA) formats. Returns nil on allocation failure.
2106
+ private static func deepCopyPixelBuffer(
2107
+ _ src: CVPixelBuffer
2108
+ ) -> CVPixelBuffer? {
2109
+ let format = CVPixelBufferGetPixelFormatType(src)
2110
+ let width = CVPixelBufferGetWidth(src)
2111
+ let height = CVPixelBufferGetHeight(src)
2112
+
2113
+ // IOSurface-backed copy so cv::imread / Vision frameworks
2114
+ // can read the buffer without re-uploading. Empty dict =
2115
+ // use default IOSurface attributes.
2116
+ let attrs: NSDictionary = [
2117
+ kCVPixelBufferIOSurfacePropertiesKey: NSDictionary(),
2118
+ ]
2119
+ var dst: CVPixelBuffer?
2120
+ let createStatus = CVPixelBufferCreate(
2121
+ kCFAllocatorDefault, width, height, format, attrs, &dst
2122
+ )
2123
+ guard createStatus == kCVReturnSuccess, let copy = dst else {
2124
+ return nil
2125
+ }
2126
+
2127
+ let srcLock = CVPixelBufferLockBaseAddress(src, .readOnly)
2128
+ let dstLock = CVPixelBufferLockBaseAddress(copy, [])
2129
+ defer {
2130
+ CVPixelBufferUnlockBaseAddress(src, .readOnly)
2131
+ CVPixelBufferUnlockBaseAddress(copy, [])
2132
+ }
2133
+ guard srcLock == kCVReturnSuccess,
2134
+ dstLock == kCVReturnSuccess else {
2135
+ return nil
2136
+ }
2137
+
2138
+ if CVPixelBufferIsPlanar(src) {
2139
+ let nPlanes = CVPixelBufferGetPlaneCount(src)
2140
+ for plane in 0..<nPlanes {
2141
+ guard
2142
+ let srcBase =
2143
+ CVPixelBufferGetBaseAddressOfPlane(src, plane),
2144
+ let dstBase =
2145
+ CVPixelBufferGetBaseAddressOfPlane(copy, plane)
2146
+ else { return nil }
2147
+ let srcStride =
2148
+ CVPixelBufferGetBytesPerRowOfPlane(src, plane)
2149
+ let dstStride =
2150
+ CVPixelBufferGetBytesPerRowOfPlane(copy, plane)
2151
+ let planeH =
2152
+ CVPixelBufferGetHeightOfPlane(src, plane)
2153
+ if srcStride == dstStride {
2154
+ memcpy(dstBase, srcBase, srcStride * planeH)
2155
+ } else {
2156
+ let rowBytes = min(srcStride, dstStride)
2157
+ for r in 0..<planeH {
2158
+ memcpy(
2159
+ dstBase.advanced(by: r * dstStride),
2160
+ srcBase.advanced(by: r * srcStride),
2161
+ rowBytes
2162
+ )
2163
+ }
2164
+ }
2165
+ }
2166
+ } else {
2167
+ // Packed (e.g. BGRA).
2168
+ guard let srcBase = CVPixelBufferGetBaseAddress(src),
2169
+ let dstBase = CVPixelBufferGetBaseAddress(copy)
2170
+ else { return nil }
2171
+ let srcStride = CVPixelBufferGetBytesPerRow(src)
2172
+ let dstStride = CVPixelBufferGetBytesPerRow(copy)
2173
+ if srcStride == dstStride {
2174
+ memcpy(dstBase, srcBase, srcStride * height)
2175
+ } else {
2176
+ let rowBytes = min(srcStride, dstStride)
2177
+ for r in 0..<height {
2178
+ memcpy(
2179
+ dstBase.advanced(by: r * dstStride),
2180
+ srcBase.advanced(by: r * srcStride),
2181
+ rowBytes
2182
+ )
2183
+ }
2184
+ }
2185
+ }
2186
+ return copy
2187
+ }
2188
+
2189
+ /// Synthesise + emit a state event for a frame the keyframe gate
2190
+ /// rejected. The native engine never sees the frame, so its own
2191
+ /// state machinery isn't invoked — but JS still wants the event
2192
+ /// so the status pill can update ("frame skipped, still 3/6").
2193
+ private func emitKeyframeRejectState(decision: KeyframeGateDecision) {
2194
+ // Pick the right outcome value for JS; defaults match the
2195
+ // intent (overlap-too-high vs cap-reached).
2196
+ let outcome: IncrementalOutcome
2197
+ switch decision.reason {
2198
+ case "max-reached": outcome = .skippedKeyframeMaxReached
2199
+ case "overlap-too-high": outcome = .skippedKeyframeOverlap
2200
+ default: outcome = .skippedKeyframeOverlap
2201
+ }
2202
+ // Re-use the previous state's pan-extent / orientation fields
2203
+ // so the band overlay doesn't flicker when a reject lands.
2204
+ //
2205
+ // V16 A2 (post-2026-05-13-14:41:57 .ips):
2206
+ // `self.lastState` is written from BOTH this method (main
2207
+ // thread, reject path) AND the engine accept-path
2208
+ // (workQueue, ~line 1908) — both under stateLock at the
2209
+ // write site, but these reads were unprotected. Pre-A2
2210
+ // the reject rate was a few per second and the race was
2211
+ // latent; A2's flow-based default raised it to ~50/s. At
2212
+ // that frequency the torn-pointer-on-class-ref race fires:
2213
+ // T0 workQueue prepares new state, about to replace
2214
+ // self.lastState
2215
+ // T1 main thread loads self.lastState — sees OLD ref
2216
+ // T2 workQueue writes new ref; ARC releases OLD
2217
+ // (refcount → 0 → freed)
2218
+ // T3 main thread's load completes; ARC tries to retain
2219
+ // OLD ref → objc_retain on freed memory →
2220
+ // EXC_BAD_ACCESS at frame 0 of emitKeyframeRejectState.
2221
+ // Fix: hold stateLock for the read. Cheap (microseconds);
2222
+ // tighter scope than wrapping the whole function so we
2223
+ // don't hold during the NotificationCenter post that
2224
+ // follows.
2225
+ let prev: IncrementalStateObject?
2226
+ let acceptedCount: Int
2227
+ stateLock.lock()
2228
+ prev = self.lastState
2229
+ acceptedCount = self.engineAcceptedCount
2230
+ stateLock.unlock()
2231
+ let overlapPercent = (decision.newContentFraction >= 0)
2232
+ ? (1.0 - decision.newContentFraction) * 100.0
2233
+ : (prev?.overlapPercent ?? -1.0)
2234
+ let state = IncrementalStateObject(
2235
+ panoramaPath: nil,
2236
+ width: 0,
2237
+ height: 0,
2238
+ acceptedCount: acceptedCount,
2239
+ outcome: outcome,
2240
+ confidence: 0,
2241
+ overlapPercent: overlapPercent,
2242
+ processingMs: 0,
2243
+ isLandscape: prev?.isLandscape ?? false,
2244
+ paintedExtent: prev?.paintedExtent ?? 0,
2245
+ panExtent: prev?.panExtent ?? 0,
2246
+ keyframeMax: decision.maxCount
2247
+ )
2248
+ stateLock.lock()
2249
+ self.lastState = state
2250
+ stateLock.unlock()
2251
+ NotificationCenter.default.post(
2252
+ name: .retailensIncrementalStateUpdate,
2253
+ object: nil,
2254
+ userInfo: state.asDictionary()
2255
+ )
2256
+ }
2257
+
2258
+ /// Read the most recent state snapshot (JS pulls this on demand).
2259
+ @objc public func currentStateDictionary() -> [String: Any]? {
2260
+ stateLock.lock()
2261
+ defer { stateLock.unlock() }
2262
+ return lastState?.asDictionary()
2263
+ }
2264
+
2265
+ // ── ARSession frame consumer hook ───────────────────────────────
2266
+
2267
+ /// Called from the ARSession delegate while the engine is active.
2268
+ /// MUST consume the pixel buffer before returning (Apple's pool
2269
+ /// reuse contract — see comments in RNSARSession).
2270
+ @objc public func consumeFrame(
2271
+ pixelBuffer: CVPixelBuffer,
2272
+ pose: RNSARFramePose
2273
+ ) {
2274
+ guard stateLock.try() else {
2275
+ // start/stop in flight — drop this frame.
2276
+ return
2277
+ }
2278
+ let hybrid = self.hybridEngine
2279
+ let slit = self.firstwinsEngine
2280
+ let isRunning = self.isRunning
2281
+ // V16 Phase 1 — capture batch-keyframe state under the lock so
2282
+ // the work-queue closure (or the synchronous reject below)
2283
+ // sees consistent ivars even if start/cancel races.
2284
+ let inBatchKeyframeMode = self.batchKeyframeMode
2285
+ let collector = self.keyframeCollector
2286
+ let rotationDegreesForBatch = self.keyframeRotationDegrees
2287
+ let exifOrientationForBatch = self.keyframeExifOrientation
2288
+
2289
+ // V13.0c.1 — diagnostic translation logging. Captures the
2290
+ // FIRST frame's world position, then logs delta from first
2291
+ // on every subsequent frame. Throttled to every 5th call
2292
+ // to keep Console.app readable. This data tells us how
2293
+ // much users physically translate during typical captures
2294
+ // before we commit to per-pixel depth correction (V13.0c.2+).
2295
+ //
2296
+ // Notes:
2297
+ // • tx,ty,tz are in ARKit world coords (metres).
2298
+ // • magnitudeM = √(Δtx² + Δty² + Δtz²) — total camera
2299
+ // displacement from first frame.
2300
+ // • If typical magnitudeM < 0.05 m (5 cm) → minimal
2301
+ // translation, NCC alone may suffice.
2302
+ // • If typical magnitudeM > 0.30 m (30 cm) → significant
2303
+ // translation, per-depth correction essential.
2304
+ if isRunning {
2305
+ self.consumeFrameCounter += 1
2306
+ if !self.hasFirstFrameTranslation {
2307
+ self.firstFrameTx = pose.tx
2308
+ self.firstFrameTy = pose.ty
2309
+ self.firstFrameTz = pose.tz
2310
+ self.hasFirstFrameTranslation = true
2311
+ // V13.0c.1.1 — FAULT-level os_log under same subsystem
2312
+ // as V13.0b-gate so logs appear under either Console.app
2313
+ // filter (process-only or subsystem).
2314
+ os_log(.fault, log: Self.diagLog,
2315
+ "[V13.0c-trans] first-frame world position tx=%.4f ty=%.4f tz=%.4f",
2316
+ pose.tx, pose.ty, pose.tz)
2317
+ } else if self.consumeFrameCounter % 5 == 0 {
2318
+ let dx = pose.tx - self.firstFrameTx
2319
+ let dy = pose.ty - self.firstFrameTy
2320
+ let dz = pose.tz - self.firstFrameTz
2321
+ let mag = sqrt(dx * dx + dy * dy + dz * dz)
2322
+ os_log(.fault, log: Self.diagLog,
2323
+ "[V13.0c-trans] #%d delta_t_world=(%+.4f,%+.4f,%+.4f) magnitude=%.4f m",
2324
+ self.consumeFrameCounter, dx, dy, dz, mag)
2325
+ }
2326
+ }
2327
+
2328
+ // V16 Phase 2 (C1 fix) — evaluate the pose-driven keyframe gate
2329
+ // BEFORE releasing stateLock.
2330
+ //
2331
+ // RCA: KeyframeGate is documented thread-unsafe (its C++ pImpl
2332
+ // holds std::optional<std::vector<Vec2>>, etc.; caller must
2333
+ // serialise). Every OTHER access already runs under stateLock —
2334
+ // `start`'s configure-gate path, `cancel`'s `keyframeGate.reset()`,
2335
+ // `markNextFrameAsLastKeyframe`'s `forceAcceptNext = true` write.
2336
+ // ONLY this read site (the AR-delegate hot path) used to run
2337
+ // OUTSIDE stateLock, which is a thread-safety hole — concurrent
2338
+ // mutation from the JS-bridge thread could corrupt the gate's
2339
+ // internal vectors mid-evaluate().
2340
+ //
2341
+ // We compute the decision under the lock, then unlock BEFORE the
2342
+ // heavier work (emitKeyframeRejectState fires a JS event; the
2343
+ // workQueue dispatch + JPEG encode + path-append happen below).
2344
+ // Holding the lock for the ~50–200 µs evaluate is negligible vs
2345
+ // the alternative (latent corruption that produced the V16
2346
+ // Phase 1b finalize crashes).
2347
+ //
2348
+ // We evaluate BEFORE the workInFlight check so a rejected frame
2349
+ // doesn't burn workQueue slots — the gate is the cheap filter,
2350
+ // the engine is the expensive one.
2351
+ var gateDecision: KeyframeGateDecision? = nil
2352
+ // V16 — eval-throttle. When flowEvalEveryNFrames > 1, run the
2353
+ // gate every Nth consumeFrame instead of every frame. Cuts
2354
+ // CPU on the AR delegate path linearly with N at the cost of
2355
+ // up to N-1 frames of acceptance latency. Doesn't change
2356
+ // WHICH frames are accepted — just when we check.
2357
+ //
2358
+ // `consumeFrameCounter` was already incremented above (line
2359
+ // ~1596) so the first frame (counter=1) is always evaluated
2360
+ // regardless of N: (1-1) % N == 0 for any N ≥ 1. Subsequent
2361
+ // evals land on counter = 1+N, 1+2N, ... — first frame
2362
+ // triggers immediately, then every Nth one after.
2363
+ let evalCadence = max(1, self.keyframeGate.flowEvalEveryNFrames)
2364
+ let cadenceFires = ((self.consumeFrameCounter - 1) % evalCadence == 0)
2365
+ let gateActive =
2366
+ isRunning
2367
+ && (hybrid != nil || slit != nil || inBatchKeyframeMode)
2368
+ && self.keyframeGate.enabled
2369
+ let shouldEvaluateGate = gateActive && cadenceFires
2370
+ // True iff the gate is active for this capture but we're
2371
+ // skipping THIS specific frame due to the throttle. In that
2372
+ // case we must also skip the workQueue save path below —
2373
+ // otherwise non-Nth frames would be unconditionally saved as
2374
+ // keyframes, which would defeat the gate entirely.
2375
+ let throttledThisFrame = gateActive && !cadenceFires
2376
+ if shouldEvaluateGate {
2377
+ let plane = RNSARSession.shared.latchedPlaneTransform()
2378
+ // V16 A2 — call the pixel-buffer-aware overload so Flow
2379
+ // strategy gets the image content. Pose strategy is
2380
+ // routed to the fast pose-only path inside the bridge,
2381
+ // so we don't pay the buffer-lock cost for Pose frames.
2382
+ gateDecision = self.keyframeGate.evaluate(
2383
+ pose: pose,
2384
+ latchedPlane: plane,
2385
+ pixelBuffer: pixelBuffer
2386
+ )
2387
+ }
2388
+ stateLock.unlock()
2389
+
2390
+ // V16 eval-throttle bail. If the gate is active but we
2391
+ // skipped evaluation for this frame, drop the entire save
2392
+ // pipeline. We emit no event and don't burn the workQueue
2393
+ // slot — the next AR-delegate frame that lands on the
2394
+ // cadence will go through normally.
2395
+ if throttledThisFrame {
2396
+ return
2397
+ }
2398
+
2399
+ // 2026-05-15 — `[V16-keyframe-decision]` per-decision diag log
2400
+ // removed. Original commit 9dd0ae9 (formerly 7664d5a pre-
2401
+ // author-rewrite) added a fault-level os_log on every gate
2402
+ // decision (accept + reject) to investigate the post-fix-7
2403
+ // bursting. Root cause was identified + fixed in fix-9/10/11
2404
+ // (see 2026-05-12-finalize-crash-investigation.md, Round 3).
2405
+ // The log was noise after that; removing it keeps the
2406
+ // fault-level Console output clean. Reject-path decisions
2407
+ // still emit via emitKeyframeRejectState() below for UI pill
2408
+ // text; accept-path decisions emit via the keyframeAccepted
2409
+ // path and the JS state subscriber.
2410
+
2411
+ // V16 Phase 1 — batch-keyframe is also a valid running mode
2412
+ // (no engine pointer, but the collector and gate are active).
2413
+ guard isRunning,
2414
+ (hybrid != nil || slit != nil || inBatchKeyframeMode)
2415
+ else { return }
2416
+
2417
+ // Surface the gate's reject decision (if any) outside the lock.
2418
+ // emitKeyframeRejectState dispatches a JS bridge event which
2419
+ // could itself acquire other locks; keeping it outside stateLock
2420
+ // is the safe call.
2421
+ if let decision = gateDecision, !decision.accept {
2422
+ self.emitKeyframeRejectState(decision: decision)
2423
+ return
2424
+ }
2425
+
2426
+ // Compute yaw + pitch from the quaternion. Convention:
2427
+ // yaw = rotation about world Y (camera turning left/right)
2428
+ // pitch = rotation about camera X (camera tilting up/down)
2429
+ let q = simd_quatf(
2430
+ ix: Float(pose.qx), iy: Float(pose.qy),
2431
+ iz: Float(pose.qz), r: Float(pose.qw)
2432
+ )
2433
+ let (yaw, pitch) = Self.yawPitch(from: q)
2434
+
2435
+ // Both FoVs from physical camera intrinsics. Passing the
2436
+ // PHYSICAL vertical FoV (vs deriving it from compose aspect
2437
+ // inside the engine) is what fixes the v1/v2 "only left-to-
2438
+ // right portrait pan responds" bug — the engine's overlap
2439
+ // gate compared world-pitch against a compose-aspect-derived
2440
+ // vertical FoV that didn't match the actual camera, so most
2441
+ // top-to-bottom pans fell outside the 30-70% window.
2442
+ let fovHRad = 2.0 * atan(Double(pose.imageWidth) / (2.0 * pose.fx))
2443
+ let fovVRad = 2.0 * atan(Double(pose.imageHeight) / (2.0 * pose.fy))
2444
+ let fovHDeg = fovHRad * 180.0 / .pi
2445
+ let fovVDeg = fovVRad * 180.0 / .pi
2446
+
2447
+ let trackingPoor = (pose.trackingState != .tracking)
2448
+
2449
+ // V11 Gap #27: dispatch the heavy pipeline (engine.ingest +
2450
+ // optional snapshot) to the work queue. Earlier versions
2451
+ // ran the full ~70 ms accept inside the AR delegate thread,
2452
+ // blocking ARKit's 16 ms inter-frame budget and causing
2453
+ // ~4-5 frames to be dropped during each accept.
2454
+ //
2455
+ // Backpressure: if the work queue is already busy with a
2456
+ // previous frame, drop this one (don't queue up — that'd
2457
+ // produce an ever-growing latency between AR-time and
2458
+ // canvas-state). CVPixelBuffer auto-retains via Swift's
2459
+ // ARC; the closure capture extends its lifetime past the
2460
+ // delegate return.
2461
+ if self.workInFlight {
2462
+ self.droppedBackpressure += 1
2463
+ return
2464
+ }
2465
+ self.workInFlight = true
2466
+
2467
+ // V16 Phase 1b.fix2 — DEEP COPY of the pixel buffer.
2468
+ //
2469
+ // Apple's contract on ARFrame.capturedImage: "valid only
2470
+ // within the scope of the captured ARFrame. To use beyond
2471
+ // that scope you must make a copy." Swift's `let pbCopy =
2472
+ // pixelBuffer` is JUST an ARC retain on the CVPixelBufferRef;
2473
+ // it does NOT extend the lifetime of the underlying IOSurface.
2474
+ // ARKit's pixel-buffer pool (~3–4 buffers) recycles slots
2475
+ // aggressively under load — long pans race the workQueue's
2476
+ // 50–100 ms JPEG encode against pool churn and randomly hit
2477
+ // a freed slot, producing the EXC_BAD_ACCESS in objc_retain
2478
+ // we saw mid-pan ("when I pan the device more").
2479
+ //
2480
+ // CVPixelBufferCreate + memcpy gives us a fully-owned copy
2481
+ // that ARC alone governs. ~1-2 ms cost on iPhone 16 Pro
2482
+ // (10 MB memcpy at memory bandwidth ~10 GB/s). Fixes the
2483
+ // crash for ALL engine paths (slit-scan / hybrid / batch-
2484
+ // keyframe) since they all dispatch via consumeFrame.
2485
+ guard let pbCopy = Self.deepCopyPixelBuffer(pixelBuffer) else {
2486
+ // Allocation failure — drop the frame. Extremely rare;
2487
+ // would only happen under genuine OOM.
2488
+ os_log(.fault, log: Self.diagLog,
2489
+ "[V16-pbcopy] CVPixelBufferCreate failed; dropping frame")
2490
+ self.workInFlight = false
2491
+ return
2492
+ }
2493
+ workQueue.async { [weak self] in
2494
+ defer { self?.workInFlight = false }
2495
+ guard let self = self else { return }
2496
+
2497
+ // V12.1 frame-leak fix: finalize/cancel may have run on
2498
+ // the JS thread between consumeFrame's isRunning check
2499
+ // and now. If isRunning is now false, this frame would
2500
+ // have been dispatched a few ms BEFORE the user released
2501
+ // the shutter — ingesting it now is exactly the
2502
+ // "phantom frame after release" the user observed. Bail
2503
+ // before touching the engine.
2504
+ self.stateLock.lock()
2505
+ let stillRunning = self.isRunning
2506
+ self.stateLock.unlock()
2507
+ guard stillRunning else { return }
2508
+
2509
+ // V16 Phase 1 — batch-keyframe path: save the buffer as
2510
+ // a JPEG via the collector, append the pose, emit a
2511
+ // notification so JS can render the thumbnail in
2512
+ // LiveFrameStrip. No incremental engine to call.
2513
+ if inBatchKeyframeMode {
2514
+ guard let coll = collector else { return }
2515
+ do {
2516
+ let record = try coll.saveKeyframe(
2517
+ pbCopy,
2518
+ rotationDegrees: rotationDegreesForBatch,
2519
+ exifOrientation: exifOrientationForBatch,
2520
+ jpegQuality: 80
2521
+ )
2522
+ self.stateLock.lock()
2523
+ self.keyframePaths.append(record.path)
2524
+ self.keyframePoses.append(pose.asDictionary())
2525
+ let count = self.keyframePaths.count
2526
+ self.stateLock.unlock()
2527
+ os_log(.fault, log: Self.diagLog,
2528
+ "[V16-batch-keyframe] saved keyframe %d → %{public}@ (%dx%d)",
2529
+ Int32(count),
2530
+ record.path,
2531
+ Int32(record.width), Int32(record.height))
2532
+ self.emitBatchKeyframeAcceptedState(
2533
+ thumbnailPath: record.path,
2534
+ keyframeIndex: Int(record.index),
2535
+ keyframeCount: count,
2536
+ keyframeMax: self.keyframeGate.maxCount,
2537
+ isLandscape: pose.imageWidth >= pose.imageHeight
2538
+ )
2539
+ } catch let err as NSError {
2540
+ os_log(.fault, log: Self.diagLog,
2541
+ "[V16-batch-keyframe] saveKeyframe failed: %{public}@",
2542
+ err.localizedDescription)
2543
+ }
2544
+ return
2545
+ }
2546
+
2547
+ // V15.0b — if a vertical plane has just been detected and
2548
+ // we haven't propagated it to the slit-scan engine yet,
2549
+ // do so now. Propagated only once per latched plane;
2550
+ // RNSARSession resets on stop().
2551
+ if !self.havePropagatedPlane,
2552
+ let plane = RNSARSession.shared.planeTransformFlat() {
2553
+ slit?.setPlaneTransformFlat(plane)
2554
+ self.havePropagatedPlane = true
2555
+ // V15.0c.4 — fault log so we can see the propagation
2556
+ // moment without rate-limit drops.
2557
+ os_log(.fault, log: Self.diagLog,
2558
+ "[V15.0b-plane] bridge propagated plane to slit-scan engine (one-shot per capture)")
2559
+ }
2560
+
2561
+ let telemetry: RLISFrameTelemetry
2562
+ if let hybrid = hybrid {
2563
+ telemetry = hybrid.ingest(
2564
+ pixelBuffer: pbCopy, qx: pose.qx, qy: pose.qy, qz: pose.qz, qw: pose.qw,
2565
+ tx: pose.tx, ty: pose.ty, tz: pose.tz,
2566
+ fx: pose.fx, fy: pose.fy, cx: pose.cx, cy: pose.cy,
2567
+ imageWidth: pose.imageWidth, imageHeight: pose.imageHeight,
2568
+ yaw: yaw, pitch: pitch,
2569
+ fovHorizDegrees: fovHDeg, fovVertDegrees: fovVDeg,
2570
+ trackingPoor: trackingPoor
2571
+ )
2572
+ } else if let slit = slit {
2573
+ // V13.0e — slit-scan engine consumes tx/ty/tz for
2574
+ // ORB-triangulation-based depth estimation and per-frame
2575
+ // translation parallax correction. Hybrid passes them
2576
+ // for API symmetry; only the slit engine uses them.
2577
+ telemetry = slit.ingest(
2578
+ pixelBuffer: pbCopy, qx: pose.qx, qy: pose.qy, qz: pose.qz, qw: pose.qw,
2579
+ tx: pose.tx, ty: pose.ty, tz: pose.tz,
2580
+ fx: pose.fx, fy: pose.fy, cx: pose.cx, cy: pose.cy,
2581
+ imageWidth: pose.imageWidth, imageHeight: pose.imageHeight,
2582
+ yaw: yaw, pitch: pitch,
2583
+ fovHorizDegrees: fovHDeg, fovVertDegrees: fovVDeg,
2584
+ trackingPoor: trackingPoor
2585
+ )
2586
+ } else {
2587
+ return
2588
+ }
2589
+
2590
+ self.processIngestResult(
2591
+ telemetry: telemetry, hybrid: hybrid, slit: slit)
2592
+ }
2593
+ }
2594
+
2595
+ /// Pulled out of consumeFrame so the work-queue closure stays
2596
+ /// readable. Same flow: build state, optionally snapshot, post
2597
+ /// notification.
2598
+ private func processIngestResult(
2599
+ telemetry: RLISFrameTelemetry,
2600
+ hybrid: OpenCVIncrementalStitcher?,
2601
+ slit: OpenCVFirstWinsCylindricalStitcher?
2602
+ ) {
2603
+ var snapshotPath: String?
2604
+ var snapW = 0, snapH = 0
2605
+ let outcome = IncrementalOutcome(rawValue: telemetry.outcome.rawValue)
2606
+ ?? .skippedTrackingPoor
2607
+
2608
+ let isAccept = (telemetry.outcome == .acceptedHigh ||
2609
+ telemetry.outcome == .acceptedMedium)
2610
+
2611
+ if isAccept {
2612
+ self.acceptsSinceSnapshot += 1
2613
+ if self.acceptsSinceSnapshot >= self.snapshotEveryNAccepts {
2614
+ self.acceptsSinceSnapshot = 0
2615
+ do {
2616
+ let snap: RLISSnapshot
2617
+ if let hybrid = hybrid {
2618
+ snap = try hybrid.snapshot(
2619
+ withJpegQuality: self.snapshotJpegQuality)
2620
+ } else {
2621
+ snap = try slit!.snapshot(
2622
+ withJpegQuality: self.snapshotJpegQuality)
2623
+ }
2624
+ snapshotPath = snap.panoramaPath
2625
+ snapW = snap.width
2626
+ snapH = snap.height
2627
+ } catch {
2628
+ // Silently dropping a snapshot is fine — next
2629
+ // accept will retry.
2630
+ }
2631
+ }
2632
+ }
2633
+
2634
+ // V16 — pass the gate's max keyframe count when the gate is
2635
+ // active so JS can render "Keyframes: n/max". Zero signals
2636
+ // "gate disabled" to the JS pill.
2637
+ let kfMax = self.keyframeGate.enabled ? self.keyframeGate.maxCount : 0
2638
+ let state = IncrementalStateObject(
2639
+ panoramaPath: snapshotPath,
2640
+ width: snapW,
2641
+ height: snapH,
2642
+ acceptedCount: hybrid?.acceptedCount ?? slit?.acceptedCount ?? 0,
2643
+ outcome: outcome,
2644
+ confidence: telemetry.confidence,
2645
+ overlapPercent: telemetry.overlapPercent,
2646
+ processingMs: telemetry.processingMs,
2647
+ isLandscape: telemetry.isLandscape,
2648
+ paintedExtent: telemetry.paintedExtent,
2649
+ panExtent: telemetry.panExtent,
2650
+ keyframeMax: kfMax
2651
+ )
2652
+ stateLock.lock()
2653
+ self.lastState = state
2654
+ stateLock.unlock()
2655
+
2656
+ // Emit always — JS may want to drive UX on rejects too.
2657
+ // NotificationCenter is thread-agnostic; the bridge converts
2658
+ // it to a main-thread RN event.
2659
+ NotificationCenter.default.post(
2660
+ name: .retailensIncrementalStateUpdate,
2661
+ object: nil,
2662
+ userInfo: state.asDictionary()
2663
+ )
2664
+ }
2665
+
2666
+ // ── Debug log file ──────────────────────────────────────────────
2667
+ //
2668
+ // iOS Console rate-limits Swift NSLog at 60 Hz, dropping most
2669
+ // output silently. File-based log captures everything, and we
2670
+ // can pull it from Xcode's device-container browser. Path:
2671
+ // <Documents>/rlis-debug.log
2672
+ private static let debugLogQueue = DispatchQueue(label: "rlis.debuglog")
2673
+ private static var debugLogPath: String = {
2674
+ let docs = NSSearchPathForDirectoriesInDomains(
2675
+ .documentDirectory, .userDomainMask, true).first ?? NSTemporaryDirectory()
2676
+ return (docs as NSString).appendingPathComponent("rlis-debug.log")
2677
+ }()
2678
+ static func fileLog(_ msg: String) {
2679
+ let line = "\(Date().timeIntervalSince1970): \(msg)\n"
2680
+ debugLogQueue.async {
2681
+ let data = line.data(using: .utf8) ?? Data()
2682
+ if let fh = FileHandle(forWritingAtPath: debugLogPath) {
2683
+ fh.seekToEndOfFile()
2684
+ fh.write(data)
2685
+ fh.closeFile()
2686
+ } else {
2687
+ try? data.write(to: URL(fileURLWithPath: debugLogPath),
2688
+ options: .atomicWrite)
2689
+ }
2690
+ }
2691
+ // Also try NSLog in case it does work occasionally
2692
+ NSLog("[RLIS-PIP] %@", msg)
2693
+ }
2694
+
2695
+ // ── Helpers ─────────────────────────────────────────────────────
2696
+
2697
+ /// Extract yaw (rotation about world Y) and pitch (rotation about
2698
+ /// camera X) from an ARKit camera quaternion. Numerically stable
2699
+ /// for camera orientations the user holds in practice — straight
2700
+ /// up/down is gimbal-locked but a shelf-audit user is never there.
2701
+ private static func yawPitch(from q: simd_quatf) -> (Double, Double) {
2702
+ // Apply the quaternion to ARKit's camera-forward vector
2703
+ // (-Z in camera frame) to get the camera-forward in world.
2704
+ // Yaw is the angle of the projection onto the X-Z plane;
2705
+ // pitch is the elevation angle.
2706
+ let forward = simd_act(q, simd_float3(0, 0, -1))
2707
+ let yaw = Double(atan2(forward.x, -forward.z))
2708
+ let pitch = Double(asin(forward.y))
2709
+ return (yaw, pitch)
2710
+ }
2711
+ }
2712
+
2713
+ // ── Bridge contract for ARSession ───────────────────────────────────
2714
+ //
2715
+ // The AR session calls into us via an @objc protocol so the dependency
2716
+ // arrow points the right way: ARSession (low-level) delivers frames
2717
+ // to a consumer it knows nothing about. The Stitcher implements the
2718
+ // protocol and registers itself.
2719
+
2720
+ @objc public protocol ARFrameConsumer: AnyObject {
2721
+ /// Called on the ARSession delegate's queue. The pixel buffer is
2722
+ /// only valid for the duration of this call (Apple's ARKit pool
2723
+ /// reuse contract); consumers must copy out before returning.
2724
+ func consumeFrame(pixelBuffer: CVPixelBuffer, pose: RNSARFramePose)
2725
+ }
2726
+
2727
+ extension IncrementalStitcher: ARFrameConsumer {}