react-native-image-stitcher 0.5.1 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (28) hide show
  1. package/CHANGELOG.md +199 -1
  2. package/android/src/main/java/io/imagestitcher/rn/CvFlowGateFrameProcessor.kt +2 -2
  3. package/android/src/main/java/io/imagestitcher/rn/IncrementalFirstwinsEngine.kt +2 -30
  4. package/android/src/main/java/io/imagestitcher/rn/IncrementalStitcher.kt +90 -368
  5. package/android/src/main/java/io/imagestitcher/rn/KeyframeGate.kt +6 -3
  6. package/android/src/main/java/io/imagestitcher/rn/RNSARCameraView.kt +17 -30
  7. package/dist/camera/Camera.d.ts +11 -27
  8. package/dist/camera/Camera.js +46 -78
  9. package/dist/index.d.ts +2 -3
  10. package/dist/index.js +10 -6
  11. package/dist/stitching/incremental.d.ts +79 -11
  12. package/dist/stitching/useFrameProcessorDriver.d.ts +7 -6
  13. package/dist/stitching/useFrameProcessorDriver.js +12 -11
  14. package/dist/stitching/useKeyframeStream.d.ts +69 -0
  15. package/dist/stitching/useKeyframeStream.js +120 -0
  16. package/ios/Sources/RNImageStitcher/IncrementalStitcher.swift +48 -208
  17. package/ios/Sources/RNImageStitcher/IncrementalStitcherBridge.m +0 -8
  18. package/ios/Sources/RNImageStitcher/IncrementalStitcherBridge.swift +6 -126
  19. package/ios/Sources/RNImageStitcher/KeyframeGateFrameProcessor.mm +6 -6
  20. package/package.json +1 -1
  21. package/src/camera/Camera.tsx +57 -106
  22. package/src/index.ts +9 -9
  23. package/src/stitching/incremental.ts +84 -11
  24. package/src/stitching/useFrameProcessorDriver.ts +12 -11
  25. package/src/stitching/useKeyframeStream.ts +127 -0
  26. package/dist/stitching/useIncrementalJSDriver.d.ts +0 -74
  27. package/dist/stitching/useIncrementalJSDriver.js +0 -220
  28. package/src/stitching/useIncrementalJSDriver.ts +0 -297
@@ -0,0 +1,69 @@
1
+ import { type AcceptedKeyframe } from './incremental';
2
+ /**
3
+ * v0.7.0 — Tier 1: subscribe to accepted-keyframe events while a
4
+ * panorama is in progress.
5
+ *
6
+ * Fires once per keyframe accepted by the stitching engine — typically
7
+ * 4-6 times per panorama, NOT per camera frame. Use for low-frequency
8
+ * per-keyframe host work such as OCR on the saved JPEG, packet
9
+ * detection, server-side analysis, or analytics.
10
+ *
11
+ * For mid-frequency frame access (sampled stream), see `useFrameStream`
12
+ * (v0.9.0+). For per-frame worklet access (~30 Hz), see
13
+ * `useFrameProcessor` (v0.8.0+).
14
+ *
15
+ * ## Engine-mode caveat (v0.7.0)
16
+ *
17
+ * Only the `batch-keyframe` engine emits these events. Live engines
18
+ * (`firstwins-rectilinear`, `hybrid`, `slitscan-*`) paint into a live
19
+ * canvas instead of saving per-accept JPEGs, and do not surface accept
20
+ * events through this channel — the hook silently does not fire when
21
+ * such an engine is active. A v0.7.1 follow-up may add live-engine
22
+ * accept emit if a real consumer needs it.
23
+ *
24
+ * ## Payload
25
+ *
26
+ * The handler receives an {@link AcceptedKeyframe}:
27
+ *
28
+ * - `jpegPath`: absolute filesystem path, no `file://` prefix. The
29
+ * JPEG is the engine's own copy under the active capture's session
30
+ * directory. It persists for the lifetime of the panorama and is
31
+ * cleaned up automatically when the panorama finalises or is
32
+ * abandoned (or via explicit `cleanupOldKeyframes`). Copy
33
+ * synchronously inside the handler if long-term retention is
34
+ * needed.
35
+ * - `pose`: rotation quaternion (always present) + optional
36
+ * translation vector (populated in AR mode; undefined in non-AR).
37
+ * - `timestamp`: milliseconds since the Unix epoch.
38
+ * - `index`: zero-based keyframe position in the current panorama.
39
+ *
40
+ * ## Lifecycle
41
+ *
42
+ * Re-subscribes on `handler` identity changes. Wrap the handler in
43
+ * `useCallback` if it closes over state or props you don't want to
44
+ * trigger re-subscription on every render.
45
+ *
46
+ * Async handlers are fire-and-forget. Rejected promises are caught
47
+ * and logged via `console.error`; no backpressure on the native side.
48
+ * Host code wanting to serialise work across keyframes should manage
49
+ * that itself (e.g., push into a queue + worker).
50
+ *
51
+ * ## Example
52
+ *
53
+ * ```tsx
54
+ * import { useCallback } from 'react';
55
+ * import { useKeyframeStream } from 'react-native-image-stitcher';
56
+ *
57
+ * function OcrPlugin() {
58
+ * useKeyframeStream(
59
+ * useCallback(async (kf) => {
60
+ * const text = await runOCR(kf.jpegPath);
61
+ * console.log(`Keyframe ${kf.index} pose=${kf.pose.rotation}:`, text);
62
+ * }, []),
63
+ * );
64
+ * return null;
65
+ * }
66
+ * ```
67
+ */
68
+ export declare function useKeyframeStream(handler: (keyframe: AcceptedKeyframe) => void | Promise<void>): void;
69
+ //# sourceMappingURL=useKeyframeStream.d.ts.map
@@ -0,0 +1,120 @@
1
+ "use strict";
2
+ // SPDX-License-Identifier: Apache-2.0
3
+ Object.defineProperty(exports, "__esModule", { value: true });
4
+ exports.useKeyframeStream = useKeyframeStream;
5
+ const react_1 = require("react");
6
+ const incremental_1 = require("./incremental");
7
+ /**
8
+ * v0.7.0 — Tier 1: subscribe to accepted-keyframe events while a
9
+ * panorama is in progress.
10
+ *
11
+ * Fires once per keyframe accepted by the stitching engine — typically
12
+ * 4-6 times per panorama, NOT per camera frame. Use for low-frequency
13
+ * per-keyframe host work such as OCR on the saved JPEG, packet
14
+ * detection, server-side analysis, or analytics.
15
+ *
16
+ * For mid-frequency frame access (sampled stream), see `useFrameStream`
17
+ * (v0.9.0+). For per-frame worklet access (~30 Hz), see
18
+ * `useFrameProcessor` (v0.8.0+).
19
+ *
20
+ * ## Engine-mode caveat (v0.7.0)
21
+ *
22
+ * Only the `batch-keyframe` engine emits these events. Live engines
23
+ * (`firstwins-rectilinear`, `hybrid`, `slitscan-*`) paint into a live
24
+ * canvas instead of saving per-accept JPEGs, and do not surface accept
25
+ * events through this channel — the hook silently does not fire when
26
+ * such an engine is active. A v0.7.1 follow-up may add live-engine
27
+ * accept emit if a real consumer needs it.
28
+ *
29
+ * ## Payload
30
+ *
31
+ * The handler receives an {@link AcceptedKeyframe}:
32
+ *
33
+ * - `jpegPath`: absolute filesystem path, no `file://` prefix. The
34
+ * JPEG is the engine's own copy under the active capture's session
35
+ * directory. It persists for the lifetime of the panorama and is
36
+ * cleaned up automatically when the panorama finalises or is
37
+ * abandoned (or via explicit `cleanupOldKeyframes`). Copy
38
+ * synchronously inside the handler if long-term retention is
39
+ * needed.
40
+ * - `pose`: rotation quaternion (always present) + optional
41
+ * translation vector (populated in AR mode; undefined in non-AR).
42
+ * - `timestamp`: milliseconds since the Unix epoch.
43
+ * - `index`: zero-based keyframe position in the current panorama.
44
+ *
45
+ * ## Lifecycle
46
+ *
47
+ * Re-subscribes on `handler` identity changes. Wrap the handler in
48
+ * `useCallback` if it closes over state or props you don't want to
49
+ * trigger re-subscription on every render.
50
+ *
51
+ * Async handlers are fire-and-forget. Rejected promises are caught
52
+ * and logged via `console.error`; no backpressure on the native side.
53
+ * Host code wanting to serialise work across keyframes should manage
54
+ * that itself (e.g., push into a queue + worker).
55
+ *
56
+ * ## Example
57
+ *
58
+ * ```tsx
59
+ * import { useCallback } from 'react';
60
+ * import { useKeyframeStream } from 'react-native-image-stitcher';
61
+ *
62
+ * function OcrPlugin() {
63
+ * useKeyframeStream(
64
+ * useCallback(async (kf) => {
65
+ * const text = await runOCR(kf.jpegPath);
66
+ * console.log(`Keyframe ${kf.index} pose=${kf.pose.rotation}:`, text);
67
+ * }, []),
68
+ * );
69
+ * return null;
70
+ * }
71
+ * ```
72
+ */
73
+ function useKeyframeStream(handler) {
74
+ (0, react_1.useEffect)(() => {
75
+ const sub = (0, incremental_1.subscribeIncrementalState)((state) => {
76
+ // The `batch-keyframe` engine emits four optional fields together
77
+ // on accept events. Non-accept emits (snapshot updates,
78
+ // refinement progress, live-engine state ticks, etc.) leave
79
+ // `batchKeyframeThumbnailPath` undefined — that's our
80
+ // accept-event sentinel.
81
+ const jpegPath = state.batchKeyframeThumbnailPath;
82
+ const index = state.batchKeyframeIndex;
83
+ if (jpegPath === undefined || index === undefined) {
84
+ return;
85
+ }
86
+ // `batchKeyframePose` + `batchKeyframeAcceptedAtMs` are
87
+ // populated alongside the path + index by the post-v0.7.0
88
+ // native emit. Defensive defaults guard against a host
89
+ // running on a slightly-older native binary (e.g., during a
90
+ // partial upgrade) — identity quaternion + `Date.now()`.
91
+ // Published v0.7.0 native always populates both.
92
+ const pose = state.batchKeyframePose ?? {
93
+ rotation: [0, 0, 0, 1],
94
+ };
95
+ const timestamp = state.batchKeyframeAcceptedAtMs ?? Date.now();
96
+ const keyframe = {
97
+ jpegPath,
98
+ pose,
99
+ timestamp,
100
+ index,
101
+ };
102
+ // Fire-and-forget. Async handler rejections are surfaced via
103
+ // console.error so they don't disappear into the void.
104
+ const result = handler(keyframe);
105
+ if (result && typeof result.catch === 'function') {
106
+ result.catch((err) => {
107
+ // eslint-disable-next-line no-console
108
+ console.error('[useKeyframeStream] handler threw:', err);
109
+ });
110
+ }
111
+ });
112
+ // `subscribeIncrementalState` returns null when the native module
113
+ // isn't linked (Expo Go, unit tests without the bridge, etc.).
114
+ // In that case we have nothing to clean up.
115
+ if (sub === null)
116
+ return;
117
+ return () => sub.remove();
118
+ }, [handler]);
119
+ }
120
+ //# sourceMappingURL=useKeyframeStream.js.map
@@ -365,12 +365,12 @@ public final class IncrementalStitcher: NSObject {
365
365
  /// F8.3 — gate for `consumeFrameFromPlugin` (the vision-camera
366
366
  /// Frame Processor producer-thread entry point). TRUE only when
367
367
  /// the current capture was started with
368
- /// `frameSourceMode == "frameProcessor"`. In any other mode
369
- /// (especially the legacy `"jsDriver"` path which feeds via
370
- /// `processFrameAtPath`), the plugin would double-feed the
371
- /// engine — pixel buffers from the producer thread + JPEG paths
372
- /// from the JS interval, racing on the same workQueue — so we
373
- /// drop the producer-thread call.
368
+ /// `frameSourceMode == "frameProcessor"`. In AR mode
369
+ /// (`frameSourceMode == "arSession"`) the plugin would double-feed
370
+ /// the engine alongside ARKit's `consumeFrame` delegate path
371
+ /// pixel buffers from the producer thread + pixel buffers from the
372
+ /// ARSession delegate, racing on the same workQueue — so we drop
373
+ /// the producer-thread call.
374
374
  ///
375
375
  /// Set under `stateLock` in `start()`, cleared under `stateLock`
376
376
  /// in `cancel()` and `finalize()`, ALSO read under `stateLock`
@@ -747,11 +747,13 @@ public final class IncrementalStitcher: NSObject {
747
747
  engineMode: String,
748
748
  captureOrientation: String = "portrait",
749
749
  configOverrides: [String: Any] = [:],
750
- // 2026-05-18 (Issue #2 regression fix): "arSession" (default,
751
- // legacy) registers as the ARSession's frame consumer.
752
- // "jsDriver" skips that registration — frames will come in
753
- // via processFrameAtPath instead. Used by iOS non-AR
754
- // captures (the vision-camera + gyro driver path).
750
+ // 2026-05-18 (Issue #2 regression fix): "arSession" (default)
751
+ // registers as the ARSession's frame consumer.
752
+ // "frameProcessor" skips that registration — frames come in
753
+ // via the vision-camera Frame Processor plugin's
754
+ // `consumeFrameFromPlugin` path instead. The pre-v0.6
755
+ // "jsDriver" mode (push frames in from JS via
756
+ // processFrameAtPath) has been removed.
755
757
  frameSourceMode: String = "arSession"
756
758
  ) {
757
759
  stateLock.lock()
@@ -947,9 +949,9 @@ public final class IncrementalStitcher: NSObject {
947
949
  }
948
950
  self.isRunning = true
949
951
  // F8.3 — enable the Frame Processor plugin's producer-thread
950
- // ingest only for the new "frameProcessor" mode. Any other
951
- // mode (arSession, jsDriver) keeps it OFF; see the ivar's
952
- // declaration comment for why.
952
+ // ingest only for the new "frameProcessor" mode. AR mode
953
+ // ("arSession") keeps it OFF; see the ivar's declaration
954
+ // comment for why.
953
955
  self.frameProcessorIngestEnabled = (frameSourceMode == "frameProcessor")
954
956
  self.snapshotJpegQuality = max(1, min(100, snapshotJpegQuality))
955
957
  self.snapshotEveryNAccepts = max(1, snapshotEveryNAccepts)
@@ -1099,9 +1101,10 @@ public final class IncrementalStitcher: NSObject {
1099
1101
  // `stateLock.try()` and corrupting
1100
1102
  // the gate's novelty math.
1101
1103
  // (Adversarial-review C1.)
1102
- // * `jsDriver` — DO NOT register. Legacy path uses
1103
- // `processFrameAtPath`; bypasses
1104
- // consumeFrame entirely.
1104
+ //
1105
+ // The pre-v0.6 `jsDriver` mode (which pushed frames via
1106
+ // `processFrameAtPath` and also skipped registration) has
1107
+ // been removed.
1105
1108
  if frameSourceMode == "arSession" {
1106
1109
  RNSARSession.shared.incrementalConsumer = self
1107
1110
  }
@@ -2149,191 +2152,6 @@ public final class IncrementalStitcher: NSObject {
2149
2152
  self.keyframeGate.acceptedCount, self.keyframeGate.maxCount)
2150
2153
  }
2151
2154
 
2152
- /// Whether the engine is currently in batch-keyframe mode.
2153
- /// Bridge reads this to decide whether the JS-driven
2154
- /// `processFrameAtPath` path can use the lightweight
2155
- /// `addBatchKeyframePath` (path-only) entry below.
2156
- @objc public var isBatchKeyframeMode: Bool {
2157
- stateLock.lock()
2158
- defer { stateLock.unlock() }
2159
- return batchKeyframeMode
2160
- }
2161
-
2162
- /// 2026-05-18 (Issue #2 v2) — JS-driver entry-point for
2163
- /// batch-keyframe captures. Mirrors Android's behaviour: the
2164
- /// caller (JS side via the IncrementalStitcherBridge) hands us a
2165
- /// JPEG file path that already exists on disk (saved by
2166
- /// vision-camera's takeSnapshot), plus a synthetic pose derived
2167
- /// from gyro integration. We:
2168
- ///
2169
- /// 1. Validate state (running + batchKeyframeMode).
2170
- /// 2. Ask the shared C++ KeyframeGate whether to accept this
2171
- /// frame. Pass `latchedPlane: nil` — non-AR captures have
2172
- /// no plane; the C++ gate falls back to a pose-only
2173
- /// angular-delta strategy. We do NOT pass a pixel buffer:
2174
- /// the Pose strategy doesn't need one, and avoiding the
2175
- /// JPEG → CVPixelBuffer round-trip dodges the iOS
2176
- /// orientation bugs that broke Issue 2 v1
2177
- /// (UIImage/CGContext Y-flip + EXIF-vs-CGImage dimension
2178
- /// mismatch — see the symptom in 2026-05-18 user report).
2179
- /// 3. If accepted, append the existing path + pose to the
2180
- /// finalize-time lists. No JPEG re-encode — the file on
2181
- /// disk IS the keyframe. `retailens::stitchFramePaths()`
2182
- /// at finalize uses `cv::imread` which natively handles
2183
- /// EXIF orientation, so the output panorama reads upright.
2184
- /// 4. Emit the same state-event the AR delegate path emits so
2185
- /// the JS live band populates identically.
2186
- ///
2187
- /// Architecture note: this is structurally parallel to Android's
2188
- /// `IncrementalStitcher.kt::processFrameAtPath`
2189
- /// `batchKeyframeMode` branch (lines 573-627). A follow-up
2190
- /// should extract the dispatch (gate-eval + path-append + emit)
2191
- /// into shared cpp/ so both platforms become 5-line wrappers
2192
- /// around a single C++ entry point.
2193
- @objc public func addBatchKeyframePath(
2194
- path: String,
2195
- pose: RNSARFramePose
2196
- ) -> Bool {
2197
- stateLock.lock()
2198
- guard self.isRunning, self.batchKeyframeMode else {
2199
- stateLock.unlock()
2200
- return false
2201
- }
2202
- stateLock.unlock()
2203
-
2204
- // 2026-05-21 (v0.3) — pixel-aware Flow-strategy evaluation.
2205
- // Pre-0.3 this was `evaluate(pose:latchedPlane:)` with no pixel
2206
- // buffer, which forced the C++ gate to silently fall back to
2207
- // Pose strategy (same bug as Android non-AR; both fixed in
2208
- // v0.3). We now decode the JPEG snapshot at `path` to a
2209
- // single-channel grayscale CVPixelBuffer and pass it through,
2210
- // so the gate's Flow strategy actually runs sparse-flow
2211
- // novelty on real image content.
2212
- //
2213
- // CGImageSource → CGContext into a OneComponent8 CVPixelBuffer.
2214
- // ~10-20 ms per snapshot on iPhone 13/16 Pro; well under the
2215
- // ~250 ms non-AR snapshot interval (~4 FPS cadence). v0.4
2216
- // will replace this path entirely by moving non-AR capture to
2217
- // vision-camera's Frame Processor API (tracked at issue #11).
2218
- let decision: KeyframeGateDecision
2219
- if let grayBuffer = Self.decodeJpegToGrayscalePixelBuffer(path: path) {
2220
- decision = self.keyframeGate.evaluate(
2221
- pose: pose,
2222
- latchedPlane: nil,
2223
- pixelBuffer: grayBuffer
2224
- )
2225
- } else {
2226
- // JPEG decode failed (corrupt file, OOM, etc.). Fall back
2227
- // to pose-only so the capture doesn't lock up — matches
2228
- // the C++ side's defensive grayData==nullptr handling
2229
- // inside evaluateWithFrame.
2230
- decision = self.keyframeGate.evaluate(
2231
- pose: pose,
2232
- latchedPlane: nil
2233
- )
2234
- }
2235
- if !decision.accept {
2236
- self.emitKeyframeRejectState(decision: decision)
2237
- return false
2238
- }
2239
-
2240
- // Append path + pose to the finalize lists. Take the lock
2241
- // briefly — these mutate state read by `finalize()`.
2242
- stateLock.lock()
2243
- self.keyframePaths.append(path)
2244
- self.keyframePoses.append(pose.asDictionary())
2245
- // 2026-05-22 (audit F2) — track first + last pose for the
2246
- // stitchMode auto-resolver. iOS parity: Android records
2247
- // these in IncrementalStitcher.kt at the same accept points.
2248
- let poseArr = [pose.tx, pose.ty, pose.tz,
2249
- pose.qx, pose.qy, pose.qz, pose.qw]
2250
- if self.batchFirstAcceptedPose == nil { self.batchFirstAcceptedPose = poseArr }
2251
- self.batchLastAcceptedPose = poseArr
2252
- let count = self.keyframePaths.count
2253
- stateLock.unlock()
2254
- os_log(.fault, log: Self.diagLog,
2255
- "[V16-batch-keyframe.js] accepted path #%d → %{public}@",
2256
- Int32(count), path)
2257
- self.emitBatchKeyframeAcceptedState(
2258
- thumbnailPath: path,
2259
- keyframeIndex: count - 1,
2260
- keyframeCount: count,
2261
- keyframeMax: self.keyframeGate.maxCount,
2262
- isLandscape: pose.imageWidth >= pose.imageHeight
2263
- )
2264
- return true
2265
- }
2266
-
2267
- /// 2026-05-21 (v0.3) — decode a JPEG file at the given path into a
2268
- /// single-channel grayscale CVPixelBuffer (`kCVPixelFormatType_-
2269
- /// OneComponent8`) suitable for feeding into the C++ KeyframeGate's
2270
- /// Flow-strategy evaluate path. The bridge's `evaluatePixelBuffer:`
2271
- /// has explicit OneComponent8 handling (added in v0.3) that reads
2272
- /// the base address as the Y plane directly, so no extra conversion
2273
- /// happens on the C++ side.
2274
- ///
2275
- /// Used by `addBatchKeyframePath` (the JS-driver non-AR path) so the
2276
- /// Flow strategy actually runs on real pixel data — pre-0.3 this
2277
- /// path called `evaluate(pose:latchedPlane:)` with no buffer and
2278
- /// the C++ side silently fell back to Pose strategy.
2279
- ///
2280
- /// Performance: ~10-20 ms for a 1920×1080 JPEG on iPhone 13/16 Pro.
2281
- /// Well under the ~250 ms non-AR snapshot interval (~4 FPS).
2282
- /// v0.4 will replace this path entirely via Frame Processor — see
2283
- /// issue #11.
2284
- ///
2285
- /// Returns nil on any failure (file missing, corrupt JPEG, OOM
2286
- /// on the CVPixelBufferCreate). Callers fall back to the
2287
- /// pose-only evaluate so the capture doesn't lock up.
2288
- private static func decodeJpegToGrayscalePixelBuffer(
2289
- path: String
2290
- ) -> CVPixelBuffer? {
2291
- let url = URL(fileURLWithPath: path)
2292
- guard let imageSource = CGImageSourceCreateWithURL(url as CFURL, nil),
2293
- let cgImage = CGImageSourceCreateImageAtIndex(imageSource, 0, nil)
2294
- else {
2295
- return nil
2296
- }
2297
- let width = cgImage.width
2298
- let height = cgImage.height
2299
-
2300
- var pixelBuffer: CVPixelBuffer?
2301
- let attrs: NSDictionary = [
2302
- kCVPixelBufferIOSurfacePropertiesKey: NSDictionary(),
2303
- ]
2304
- let status = CVPixelBufferCreate(
2305
- kCFAllocatorDefault,
2306
- width, height,
2307
- kCVPixelFormatType_OneComponent8,
2308
- attrs,
2309
- &pixelBuffer
2310
- )
2311
- guard status == kCVReturnSuccess, let buffer = pixelBuffer else {
2312
- return nil
2313
- }
2314
-
2315
- CVPixelBufferLockBaseAddress(buffer, [])
2316
- defer { CVPixelBufferUnlockBaseAddress(buffer, []) }
2317
- guard let baseAddress = CVPixelBufferGetBaseAddress(buffer) else {
2318
- return nil
2319
- }
2320
- let bytesPerRow = CVPixelBufferGetBytesPerRow(buffer)
2321
- let colorSpace = CGColorSpaceCreateDeviceGray()
2322
- guard let context = CGContext(
2323
- data: baseAddress,
2324
- width: width,
2325
- height: height,
2326
- bitsPerComponent: 8,
2327
- bytesPerRow: bytesPerRow,
2328
- space: colorSpace,
2329
- bitmapInfo: CGImageAlphaInfo.none.rawValue
2330
- ) else {
2331
- return nil
2332
- }
2333
- context.draw(cgImage, in: CGRect(x: 0, y: 0, width: width, height: height))
2334
- return buffer
2335
- }
2336
-
2337
2155
  /// V16 Phase 1 — emit a state event when a batch-keyframe is
2338
2156
  /// saved. Carries the on-disk thumbnail path so JS can render it
2339
2157
  /// in LiveFrameStrip + advance the "Keyframes: N/M" pill.
@@ -2342,7 +2160,13 @@ public final class IncrementalStitcher: NSObject {
2342
2160
  keyframeIndex: Int,
2343
2161
  keyframeCount: Int,
2344
2162
  keyframeMax: Int,
2345
- isLandscape: Bool
2163
+ isLandscape: Bool,
2164
+ // v0.7.0 — Tier 1 hook fields. `pose` is the AR pose at the
2165
+ // accept moment (gyro-synthesised in non-AR mode — translation
2166
+ // will read as ~zeros). `acceptedAtMs` is wall-clock ms since
2167
+ // Unix epoch; matches `Date.now()` on the JS side.
2168
+ pose: RNSARFramePose,
2169
+ acceptedAtMs: Double
2346
2170
  ) {
2347
2171
  let state = IncrementalStateObject(
2348
2172
  panoramaPath: nil,
@@ -2366,6 +2190,16 @@ public final class IncrementalStitcher: NSObject {
2366
2190
  // carry — JS reads these directly from the userInfo blob.
2367
2191
  dict["batchKeyframeThumbnailPath"] = thumbnailPath
2368
2192
  dict["batchKeyframeIndex"] = keyframeIndex
2193
+ // v0.7.0 — Tier 1 hook (useKeyframeStream) reads these. See
2194
+ // `AcceptedKeyframe` in src/stitching/incremental.ts. Translation
2195
+ // is always emitted; AR mode populates it from the camera
2196
+ // transform, non-AR mode reads ~zeros (gyro-only, no spatial
2197
+ // anchor).
2198
+ dict["batchKeyframePose"] = [
2199
+ "rotation": [pose.qx, pose.qy, pose.qz, pose.qw],
2200
+ "translation": [pose.tx, pose.ty, pose.tz],
2201
+ ] as [String: Any]
2202
+ dict["batchKeyframeAcceptedAtMs"] = acceptedAtMs
2369
2203
  NotificationCenter.default.post(
2370
2204
  name: .retailensIncrementalStateUpdate,
2371
2205
  object: nil,
@@ -2819,7 +2653,12 @@ public final class IncrementalStitcher: NSObject {
2819
2653
  keyframeIndex: Int(record.index),
2820
2654
  keyframeCount: count,
2821
2655
  keyframeMax: self.keyframeGate.maxCount,
2822
- isLandscape: pose.imageWidth >= pose.imageHeight
2656
+ isLandscape: pose.imageWidth >= pose.imageHeight,
2657
+ // v0.7.0 — Tier 1 hook: pose snapshot + accept
2658
+ // timestamp threaded through to JS via the
2659
+ // existing state-update channel.
2660
+ pose: pose,
2661
+ acceptedAtMs: Date().timeIntervalSince1970 * 1000
2823
2662
  )
2824
2663
  } catch let err as NSError {
2825
2664
  os_log(.fault, log: Self.diagLog,
@@ -3119,14 +2958,15 @@ extension IncrementalStitcher: ARFrameConsumer {}
3119
2958
  // * `pixelBuffer` from `frame.buffer` (vision-camera YUV biplanar)
3120
2959
  // * `tx`/`ty`/`tz` = 0 (no AR translation; gyro only gives rotation)
3121
2960
  // * `qx,qy,qz,qw` from JS-thread gyro-integrated yaw+pitch (synthesised
3122
- // as `q = q_yaw * q_pitch` same convention as
3123
- // `useIncrementalJSDriver`'s pose synthesis)
2961
+ // as `q = q_yaw * q_pitch`). `useFrameProcessorDriver` and (pre-v0.6)
2962
+ // `useIncrementalJSDriver` both produced quaternions with this layout.
3124
2963
  // * `fx`/`fy` from frame dims + assumed FoV
3125
2964
  // * `cx`/`cy` at image centre
3126
2965
  // * `trackingStateRaw = 2` (= `.tracking`) — non-AR captures don't have
3127
2966
  // a real ARKit tracking-quality signal; reporting `.tracking` keeps
3128
- // the engine's `trackingPoor` path inactive, matching the legacy
3129
- // `useIncrementalJSDriver` contract.
2967
+ // the engine's `trackingPoor` path inactive. Both the v0.6+
2968
+ // `useFrameProcessorDriver` and the pre-v0.6 `useIncrementalJSDriver`
2969
+ // follow(ed) this contract.
3130
2970
  extension IncrementalStitcher {
3131
2971
  @objc public func consumeFrameFromPlugin(
3132
2972
  pixelBuffer: CVPixelBuffer,
@@ -65,14 +65,6 @@ RCT_EXTERN_METHOD(refinePanorama:(NSDictionary *)options
65
65
  resolver:(RCTPromiseResolveBlock)resolver
66
66
  rejecter:(RCTPromiseRejectBlock)rejecter)
67
67
 
68
- // 2026-05-17 (Issue #2) — iOS non-AR frame ingestion. JS-side driver
69
- // hands snapshot file paths + gyro-derived pose to the engine so the
70
- // live band populates in non-AR mode (parity with Android). See JS
71
- // `useIncrementalVisionCameraDriver`.
72
- RCT_EXTERN_METHOD(processFrameAtPath:(NSDictionary *)options
73
- resolver:(RCTPromiseResolveBlock)resolver
74
- rejecter:(RCTPromiseRejectBlock)rejecter)
75
-
76
68
  // 2026-05-18 (Iss 3) — keyframe storage management. cleanupKeyframes
77
69
  // GCs stale per-session directories under Library/Application Support/
78
70
  // Captures; getKeyframeDir returns the active capture's session dir.