react-native-image-stitcher 0.5.1 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +199 -1
- package/android/src/main/java/io/imagestitcher/rn/CvFlowGateFrameProcessor.kt +2 -2
- package/android/src/main/java/io/imagestitcher/rn/IncrementalFirstwinsEngine.kt +2 -30
- package/android/src/main/java/io/imagestitcher/rn/IncrementalStitcher.kt +90 -368
- package/android/src/main/java/io/imagestitcher/rn/KeyframeGate.kt +6 -3
- package/android/src/main/java/io/imagestitcher/rn/RNSARCameraView.kt +17 -30
- package/dist/camera/Camera.d.ts +11 -27
- package/dist/camera/Camera.js +46 -78
- package/dist/index.d.ts +2 -3
- package/dist/index.js +10 -6
- package/dist/stitching/incremental.d.ts +79 -11
- package/dist/stitching/useFrameProcessorDriver.d.ts +7 -6
- package/dist/stitching/useFrameProcessorDriver.js +12 -11
- package/dist/stitching/useKeyframeStream.d.ts +69 -0
- package/dist/stitching/useKeyframeStream.js +120 -0
- package/ios/Sources/RNImageStitcher/IncrementalStitcher.swift +48 -208
- package/ios/Sources/RNImageStitcher/IncrementalStitcherBridge.m +0 -8
- package/ios/Sources/RNImageStitcher/IncrementalStitcherBridge.swift +6 -126
- package/ios/Sources/RNImageStitcher/KeyframeGateFrameProcessor.mm +6 -6
- package/package.json +1 -1
- package/src/camera/Camera.tsx +57 -106
- package/src/index.ts +9 -9
- package/src/stitching/incremental.ts +84 -11
- package/src/stitching/useFrameProcessorDriver.ts +12 -11
- package/src/stitching/useKeyframeStream.ts +127 -0
- package/dist/stitching/useIncrementalJSDriver.d.ts +0 -74
- package/dist/stitching/useIncrementalJSDriver.js +0 -220
- package/src/stitching/useIncrementalJSDriver.ts +0 -297
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
import { type AcceptedKeyframe } from './incremental';
|
|
2
|
+
/**
|
|
3
|
+
* v0.7.0 — Tier 1: subscribe to accepted-keyframe events while a
|
|
4
|
+
* panorama is in progress.
|
|
5
|
+
*
|
|
6
|
+
* Fires once per keyframe accepted by the stitching engine — typically
|
|
7
|
+
* 4-6 times per panorama, NOT per camera frame. Use for low-frequency
|
|
8
|
+
* per-keyframe host work such as OCR on the saved JPEG, packet
|
|
9
|
+
* detection, server-side analysis, or analytics.
|
|
10
|
+
*
|
|
11
|
+
* For mid-frequency frame access (sampled stream), see `useFrameStream`
|
|
12
|
+
* (v0.9.0+). For per-frame worklet access (~30 Hz), see
|
|
13
|
+
* `useFrameProcessor` (v0.8.0+).
|
|
14
|
+
*
|
|
15
|
+
* ## Engine-mode caveat (v0.7.0)
|
|
16
|
+
*
|
|
17
|
+
* Only the `batch-keyframe` engine emits these events. Live engines
|
|
18
|
+
* (`firstwins-rectilinear`, `hybrid`, `slitscan-*`) paint into a live
|
|
19
|
+
* canvas instead of saving per-accept JPEGs, and do not surface accept
|
|
20
|
+
* events through this channel — the hook silently does not fire when
|
|
21
|
+
* such an engine is active. A v0.7.1 follow-up may add live-engine
|
|
22
|
+
* accept emit if a real consumer needs it.
|
|
23
|
+
*
|
|
24
|
+
* ## Payload
|
|
25
|
+
*
|
|
26
|
+
* The handler receives an {@link AcceptedKeyframe}:
|
|
27
|
+
*
|
|
28
|
+
* - `jpegPath`: absolute filesystem path, no `file://` prefix. The
|
|
29
|
+
* JPEG is the engine's own copy under the active capture's session
|
|
30
|
+
* directory. It persists for the lifetime of the panorama and is
|
|
31
|
+
* cleaned up automatically when the panorama finalises or is
|
|
32
|
+
* abandoned (or via explicit `cleanupOldKeyframes`). Copy
|
|
33
|
+
* synchronously inside the handler if long-term retention is
|
|
34
|
+
* needed.
|
|
35
|
+
* - `pose`: rotation quaternion (always present) + optional
|
|
36
|
+
* translation vector (populated in AR mode; undefined in non-AR).
|
|
37
|
+
* - `timestamp`: milliseconds since the Unix epoch.
|
|
38
|
+
* - `index`: zero-based keyframe position in the current panorama.
|
|
39
|
+
*
|
|
40
|
+
* ## Lifecycle
|
|
41
|
+
*
|
|
42
|
+
* Re-subscribes on `handler` identity changes. Wrap the handler in
|
|
43
|
+
* `useCallback` if it closes over state or props you don't want to
|
|
44
|
+
* trigger re-subscription on every render.
|
|
45
|
+
*
|
|
46
|
+
* Async handlers are fire-and-forget. Rejected promises are caught
|
|
47
|
+
* and logged via `console.error`; no backpressure on the native side.
|
|
48
|
+
* Host code wanting to serialise work across keyframes should manage
|
|
49
|
+
* that itself (e.g., push into a queue + worker).
|
|
50
|
+
*
|
|
51
|
+
* ## Example
|
|
52
|
+
*
|
|
53
|
+
* ```tsx
|
|
54
|
+
* import { useCallback } from 'react';
|
|
55
|
+
* import { useKeyframeStream } from 'react-native-image-stitcher';
|
|
56
|
+
*
|
|
57
|
+
* function OcrPlugin() {
|
|
58
|
+
* useKeyframeStream(
|
|
59
|
+
* useCallback(async (kf) => {
|
|
60
|
+
* const text = await runOCR(kf.jpegPath);
|
|
61
|
+
* console.log(`Keyframe ${kf.index} pose=${kf.pose.rotation}:`, text);
|
|
62
|
+
* }, []),
|
|
63
|
+
* );
|
|
64
|
+
* return null;
|
|
65
|
+
* }
|
|
66
|
+
* ```
|
|
67
|
+
*/
|
|
68
|
+
export declare function useKeyframeStream(handler: (keyframe: AcceptedKeyframe) => void | Promise<void>): void;
|
|
69
|
+
//# sourceMappingURL=useKeyframeStream.d.ts.map
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
// SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
4
|
+
exports.useKeyframeStream = useKeyframeStream;
|
|
5
|
+
const react_1 = require("react");
|
|
6
|
+
const incremental_1 = require("./incremental");
|
|
7
|
+
/**
|
|
8
|
+
* v0.7.0 — Tier 1: subscribe to accepted-keyframe events while a
|
|
9
|
+
* panorama is in progress.
|
|
10
|
+
*
|
|
11
|
+
* Fires once per keyframe accepted by the stitching engine — typically
|
|
12
|
+
* 4-6 times per panorama, NOT per camera frame. Use for low-frequency
|
|
13
|
+
* per-keyframe host work such as OCR on the saved JPEG, packet
|
|
14
|
+
* detection, server-side analysis, or analytics.
|
|
15
|
+
*
|
|
16
|
+
* For mid-frequency frame access (sampled stream), see `useFrameStream`
|
|
17
|
+
* (v0.9.0+). For per-frame worklet access (~30 Hz), see
|
|
18
|
+
* `useFrameProcessor` (v0.8.0+).
|
|
19
|
+
*
|
|
20
|
+
* ## Engine-mode caveat (v0.7.0)
|
|
21
|
+
*
|
|
22
|
+
* Only the `batch-keyframe` engine emits these events. Live engines
|
|
23
|
+
* (`firstwins-rectilinear`, `hybrid`, `slitscan-*`) paint into a live
|
|
24
|
+
* canvas instead of saving per-accept JPEGs, and do not surface accept
|
|
25
|
+
* events through this channel — the hook silently does not fire when
|
|
26
|
+
* such an engine is active. A v0.7.1 follow-up may add live-engine
|
|
27
|
+
* accept emit if a real consumer needs it.
|
|
28
|
+
*
|
|
29
|
+
* ## Payload
|
|
30
|
+
*
|
|
31
|
+
* The handler receives an {@link AcceptedKeyframe}:
|
|
32
|
+
*
|
|
33
|
+
* - `jpegPath`: absolute filesystem path, no `file://` prefix. The
|
|
34
|
+
* JPEG is the engine's own copy under the active capture's session
|
|
35
|
+
* directory. It persists for the lifetime of the panorama and is
|
|
36
|
+
* cleaned up automatically when the panorama finalises or is
|
|
37
|
+
* abandoned (or via explicit `cleanupOldKeyframes`). Copy
|
|
38
|
+
* synchronously inside the handler if long-term retention is
|
|
39
|
+
* needed.
|
|
40
|
+
* - `pose`: rotation quaternion (always present) + optional
|
|
41
|
+
* translation vector (populated in AR mode; undefined in non-AR).
|
|
42
|
+
* - `timestamp`: milliseconds since the Unix epoch.
|
|
43
|
+
* - `index`: zero-based keyframe position in the current panorama.
|
|
44
|
+
*
|
|
45
|
+
* ## Lifecycle
|
|
46
|
+
*
|
|
47
|
+
* Re-subscribes on `handler` identity changes. Wrap the handler in
|
|
48
|
+
* `useCallback` if it closes over state or props you don't want to
|
|
49
|
+
* trigger re-subscription on every render.
|
|
50
|
+
*
|
|
51
|
+
* Async handlers are fire-and-forget. Rejected promises are caught
|
|
52
|
+
* and logged via `console.error`; no backpressure on the native side.
|
|
53
|
+
* Host code wanting to serialise work across keyframes should manage
|
|
54
|
+
* that itself (e.g., push into a queue + worker).
|
|
55
|
+
*
|
|
56
|
+
* ## Example
|
|
57
|
+
*
|
|
58
|
+
* ```tsx
|
|
59
|
+
* import { useCallback } from 'react';
|
|
60
|
+
* import { useKeyframeStream } from 'react-native-image-stitcher';
|
|
61
|
+
*
|
|
62
|
+
* function OcrPlugin() {
|
|
63
|
+
* useKeyframeStream(
|
|
64
|
+
* useCallback(async (kf) => {
|
|
65
|
+
* const text = await runOCR(kf.jpegPath);
|
|
66
|
+
* console.log(`Keyframe ${kf.index} pose=${kf.pose.rotation}:`, text);
|
|
67
|
+
* }, []),
|
|
68
|
+
* );
|
|
69
|
+
* return null;
|
|
70
|
+
* }
|
|
71
|
+
* ```
|
|
72
|
+
*/
|
|
73
|
+
function useKeyframeStream(handler) {
|
|
74
|
+
(0, react_1.useEffect)(() => {
|
|
75
|
+
const sub = (0, incremental_1.subscribeIncrementalState)((state) => {
|
|
76
|
+
// The `batch-keyframe` engine emits four optional fields together
|
|
77
|
+
// on accept events. Non-accept emits (snapshot updates,
|
|
78
|
+
// refinement progress, live-engine state ticks, etc.) leave
|
|
79
|
+
// `batchKeyframeThumbnailPath` undefined — that's our
|
|
80
|
+
// accept-event sentinel.
|
|
81
|
+
const jpegPath = state.batchKeyframeThumbnailPath;
|
|
82
|
+
const index = state.batchKeyframeIndex;
|
|
83
|
+
if (jpegPath === undefined || index === undefined) {
|
|
84
|
+
return;
|
|
85
|
+
}
|
|
86
|
+
// `batchKeyframePose` + `batchKeyframeAcceptedAtMs` are
|
|
87
|
+
// populated alongside the path + index by the post-v0.7.0
|
|
88
|
+
// native emit. Defensive defaults guard against a host
|
|
89
|
+
// running on a slightly-older native binary (e.g., during a
|
|
90
|
+
// partial upgrade) — identity quaternion + `Date.now()`.
|
|
91
|
+
// Published v0.7.0 native always populates both.
|
|
92
|
+
const pose = state.batchKeyframePose ?? {
|
|
93
|
+
rotation: [0, 0, 0, 1],
|
|
94
|
+
};
|
|
95
|
+
const timestamp = state.batchKeyframeAcceptedAtMs ?? Date.now();
|
|
96
|
+
const keyframe = {
|
|
97
|
+
jpegPath,
|
|
98
|
+
pose,
|
|
99
|
+
timestamp,
|
|
100
|
+
index,
|
|
101
|
+
};
|
|
102
|
+
// Fire-and-forget. Async handler rejections are surfaced via
|
|
103
|
+
// console.error so they don't disappear into the void.
|
|
104
|
+
const result = handler(keyframe);
|
|
105
|
+
if (result && typeof result.catch === 'function') {
|
|
106
|
+
result.catch((err) => {
|
|
107
|
+
// eslint-disable-next-line no-console
|
|
108
|
+
console.error('[useKeyframeStream] handler threw:', err);
|
|
109
|
+
});
|
|
110
|
+
}
|
|
111
|
+
});
|
|
112
|
+
// `subscribeIncrementalState` returns null when the native module
|
|
113
|
+
// isn't linked (Expo Go, unit tests without the bridge, etc.).
|
|
114
|
+
// In that case we have nothing to clean up.
|
|
115
|
+
if (sub === null)
|
|
116
|
+
return;
|
|
117
|
+
return () => sub.remove();
|
|
118
|
+
}, [handler]);
|
|
119
|
+
}
|
|
120
|
+
//# sourceMappingURL=useKeyframeStream.js.map
|
|
@@ -365,12 +365,12 @@ public final class IncrementalStitcher: NSObject {
|
|
|
365
365
|
/// F8.3 — gate for `consumeFrameFromPlugin` (the vision-camera
|
|
366
366
|
/// Frame Processor producer-thread entry point). TRUE only when
|
|
367
367
|
/// the current capture was started with
|
|
368
|
-
/// `frameSourceMode == "frameProcessor"`. In
|
|
369
|
-
/// (
|
|
370
|
-
/// `
|
|
371
|
-
///
|
|
372
|
-
///
|
|
373
|
-
///
|
|
368
|
+
/// `frameSourceMode == "frameProcessor"`. In AR mode
|
|
369
|
+
/// (`frameSourceMode == "arSession"`) the plugin would double-feed
|
|
370
|
+
/// the engine alongside ARKit's `consumeFrame` delegate path —
|
|
371
|
+
/// pixel buffers from the producer thread + pixel buffers from the
|
|
372
|
+
/// ARSession delegate, racing on the same workQueue — so we drop
|
|
373
|
+
/// the producer-thread call.
|
|
374
374
|
///
|
|
375
375
|
/// Set under `stateLock` in `start()`, cleared under `stateLock`
|
|
376
376
|
/// in `cancel()` and `finalize()`, ALSO read under `stateLock`
|
|
@@ -747,11 +747,13 @@ public final class IncrementalStitcher: NSObject {
|
|
|
747
747
|
engineMode: String,
|
|
748
748
|
captureOrientation: String = "portrait",
|
|
749
749
|
configOverrides: [String: Any] = [:],
|
|
750
|
-
// 2026-05-18 (Issue #2 regression fix): "arSession" (default
|
|
751
|
-
//
|
|
752
|
-
// "
|
|
753
|
-
// via
|
|
754
|
-
//
|
|
750
|
+
// 2026-05-18 (Issue #2 regression fix): "arSession" (default)
|
|
751
|
+
// registers as the ARSession's frame consumer.
|
|
752
|
+
// "frameProcessor" skips that registration — frames come in
|
|
753
|
+
// via the vision-camera Frame Processor plugin's
|
|
754
|
+
// `consumeFrameFromPlugin` path instead. The pre-v0.6
|
|
755
|
+
// "jsDriver" mode (push frames in from JS via
|
|
756
|
+
// processFrameAtPath) has been removed.
|
|
755
757
|
frameSourceMode: String = "arSession"
|
|
756
758
|
) {
|
|
757
759
|
stateLock.lock()
|
|
@@ -947,9 +949,9 @@ public final class IncrementalStitcher: NSObject {
|
|
|
947
949
|
}
|
|
948
950
|
self.isRunning = true
|
|
949
951
|
// F8.3 — enable the Frame Processor plugin's producer-thread
|
|
950
|
-
// ingest only for the new "frameProcessor" mode.
|
|
951
|
-
//
|
|
952
|
-
//
|
|
952
|
+
// ingest only for the new "frameProcessor" mode. AR mode
|
|
953
|
+
// ("arSession") keeps it OFF; see the ivar's declaration
|
|
954
|
+
// comment for why.
|
|
953
955
|
self.frameProcessorIngestEnabled = (frameSourceMode == "frameProcessor")
|
|
954
956
|
self.snapshotJpegQuality = max(1, min(100, snapshotJpegQuality))
|
|
955
957
|
self.snapshotEveryNAccepts = max(1, snapshotEveryNAccepts)
|
|
@@ -1099,9 +1101,10 @@ public final class IncrementalStitcher: NSObject {
|
|
|
1099
1101
|
// `stateLock.try()` and corrupting
|
|
1100
1102
|
// the gate's novelty math.
|
|
1101
1103
|
// (Adversarial-review C1.)
|
|
1102
|
-
//
|
|
1103
|
-
//
|
|
1104
|
-
//
|
|
1104
|
+
//
|
|
1105
|
+
// The pre-v0.6 `jsDriver` mode (which pushed frames via
|
|
1106
|
+
// `processFrameAtPath` and also skipped registration) has
|
|
1107
|
+
// been removed.
|
|
1105
1108
|
if frameSourceMode == "arSession" {
|
|
1106
1109
|
RNSARSession.shared.incrementalConsumer = self
|
|
1107
1110
|
}
|
|
@@ -2149,191 +2152,6 @@ public final class IncrementalStitcher: NSObject {
|
|
|
2149
2152
|
self.keyframeGate.acceptedCount, self.keyframeGate.maxCount)
|
|
2150
2153
|
}
|
|
2151
2154
|
|
|
2152
|
-
/// Whether the engine is currently in batch-keyframe mode.
|
|
2153
|
-
/// Bridge reads this to decide whether the JS-driven
|
|
2154
|
-
/// `processFrameAtPath` path can use the lightweight
|
|
2155
|
-
/// `addBatchKeyframePath` (path-only) entry below.
|
|
2156
|
-
@objc public var isBatchKeyframeMode: Bool {
|
|
2157
|
-
stateLock.lock()
|
|
2158
|
-
defer { stateLock.unlock() }
|
|
2159
|
-
return batchKeyframeMode
|
|
2160
|
-
}
|
|
2161
|
-
|
|
2162
|
-
/// 2026-05-18 (Issue #2 v2) — JS-driver entry-point for
|
|
2163
|
-
/// batch-keyframe captures. Mirrors Android's behaviour: the
|
|
2164
|
-
/// caller (JS side via the IncrementalStitcherBridge) hands us a
|
|
2165
|
-
/// JPEG file path that already exists on disk (saved by
|
|
2166
|
-
/// vision-camera's takeSnapshot), plus a synthetic pose derived
|
|
2167
|
-
/// from gyro integration. We:
|
|
2168
|
-
///
|
|
2169
|
-
/// 1. Validate state (running + batchKeyframeMode).
|
|
2170
|
-
/// 2. Ask the shared C++ KeyframeGate whether to accept this
|
|
2171
|
-
/// frame. Pass `latchedPlane: nil` — non-AR captures have
|
|
2172
|
-
/// no plane; the C++ gate falls back to a pose-only
|
|
2173
|
-
/// angular-delta strategy. We do NOT pass a pixel buffer:
|
|
2174
|
-
/// the Pose strategy doesn't need one, and avoiding the
|
|
2175
|
-
/// JPEG → CVPixelBuffer round-trip dodges the iOS
|
|
2176
|
-
/// orientation bugs that broke Issue 2 v1
|
|
2177
|
-
/// (UIImage/CGContext Y-flip + EXIF-vs-CGImage dimension
|
|
2178
|
-
/// mismatch — see the symptom in 2026-05-18 user report).
|
|
2179
|
-
/// 3. If accepted, append the existing path + pose to the
|
|
2180
|
-
/// finalize-time lists. No JPEG re-encode — the file on
|
|
2181
|
-
/// disk IS the keyframe. `retailens::stitchFramePaths()`
|
|
2182
|
-
/// at finalize uses `cv::imread` which natively handles
|
|
2183
|
-
/// EXIF orientation, so the output panorama reads upright.
|
|
2184
|
-
/// 4. Emit the same state-event the AR delegate path emits so
|
|
2185
|
-
/// the JS live band populates identically.
|
|
2186
|
-
///
|
|
2187
|
-
/// Architecture note: this is structurally parallel to Android's
|
|
2188
|
-
/// `IncrementalStitcher.kt::processFrameAtPath`
|
|
2189
|
-
/// `batchKeyframeMode` branch (lines 573-627). A follow-up
|
|
2190
|
-
/// should extract the dispatch (gate-eval + path-append + emit)
|
|
2191
|
-
/// into shared cpp/ so both platforms become 5-line wrappers
|
|
2192
|
-
/// around a single C++ entry point.
|
|
2193
|
-
@objc public func addBatchKeyframePath(
|
|
2194
|
-
path: String,
|
|
2195
|
-
pose: RNSARFramePose
|
|
2196
|
-
) -> Bool {
|
|
2197
|
-
stateLock.lock()
|
|
2198
|
-
guard self.isRunning, self.batchKeyframeMode else {
|
|
2199
|
-
stateLock.unlock()
|
|
2200
|
-
return false
|
|
2201
|
-
}
|
|
2202
|
-
stateLock.unlock()
|
|
2203
|
-
|
|
2204
|
-
// 2026-05-21 (v0.3) — pixel-aware Flow-strategy evaluation.
|
|
2205
|
-
// Pre-0.3 this was `evaluate(pose:latchedPlane:)` with no pixel
|
|
2206
|
-
// buffer, which forced the C++ gate to silently fall back to
|
|
2207
|
-
// Pose strategy (same bug as Android non-AR; both fixed in
|
|
2208
|
-
// v0.3). We now decode the JPEG snapshot at `path` to a
|
|
2209
|
-
// single-channel grayscale CVPixelBuffer and pass it through,
|
|
2210
|
-
// so the gate's Flow strategy actually runs sparse-flow
|
|
2211
|
-
// novelty on real image content.
|
|
2212
|
-
//
|
|
2213
|
-
// CGImageSource → CGContext into a OneComponent8 CVPixelBuffer.
|
|
2214
|
-
// ~10-20 ms per snapshot on iPhone 13/16 Pro; well under the
|
|
2215
|
-
// ~250 ms non-AR snapshot interval (~4 FPS cadence). v0.4
|
|
2216
|
-
// will replace this path entirely by moving non-AR capture to
|
|
2217
|
-
// vision-camera's Frame Processor API (tracked at issue #11).
|
|
2218
|
-
let decision: KeyframeGateDecision
|
|
2219
|
-
if let grayBuffer = Self.decodeJpegToGrayscalePixelBuffer(path: path) {
|
|
2220
|
-
decision = self.keyframeGate.evaluate(
|
|
2221
|
-
pose: pose,
|
|
2222
|
-
latchedPlane: nil,
|
|
2223
|
-
pixelBuffer: grayBuffer
|
|
2224
|
-
)
|
|
2225
|
-
} else {
|
|
2226
|
-
// JPEG decode failed (corrupt file, OOM, etc.). Fall back
|
|
2227
|
-
// to pose-only so the capture doesn't lock up — matches
|
|
2228
|
-
// the C++ side's defensive grayData==nullptr handling
|
|
2229
|
-
// inside evaluateWithFrame.
|
|
2230
|
-
decision = self.keyframeGate.evaluate(
|
|
2231
|
-
pose: pose,
|
|
2232
|
-
latchedPlane: nil
|
|
2233
|
-
)
|
|
2234
|
-
}
|
|
2235
|
-
if !decision.accept {
|
|
2236
|
-
self.emitKeyframeRejectState(decision: decision)
|
|
2237
|
-
return false
|
|
2238
|
-
}
|
|
2239
|
-
|
|
2240
|
-
// Append path + pose to the finalize lists. Take the lock
|
|
2241
|
-
// briefly — these mutate state read by `finalize()`.
|
|
2242
|
-
stateLock.lock()
|
|
2243
|
-
self.keyframePaths.append(path)
|
|
2244
|
-
self.keyframePoses.append(pose.asDictionary())
|
|
2245
|
-
// 2026-05-22 (audit F2) — track first + last pose for the
|
|
2246
|
-
// stitchMode auto-resolver. iOS parity: Android records
|
|
2247
|
-
// these in IncrementalStitcher.kt at the same accept points.
|
|
2248
|
-
let poseArr = [pose.tx, pose.ty, pose.tz,
|
|
2249
|
-
pose.qx, pose.qy, pose.qz, pose.qw]
|
|
2250
|
-
if self.batchFirstAcceptedPose == nil { self.batchFirstAcceptedPose = poseArr }
|
|
2251
|
-
self.batchLastAcceptedPose = poseArr
|
|
2252
|
-
let count = self.keyframePaths.count
|
|
2253
|
-
stateLock.unlock()
|
|
2254
|
-
os_log(.fault, log: Self.diagLog,
|
|
2255
|
-
"[V16-batch-keyframe.js] accepted path #%d → %{public}@",
|
|
2256
|
-
Int32(count), path)
|
|
2257
|
-
self.emitBatchKeyframeAcceptedState(
|
|
2258
|
-
thumbnailPath: path,
|
|
2259
|
-
keyframeIndex: count - 1,
|
|
2260
|
-
keyframeCount: count,
|
|
2261
|
-
keyframeMax: self.keyframeGate.maxCount,
|
|
2262
|
-
isLandscape: pose.imageWidth >= pose.imageHeight
|
|
2263
|
-
)
|
|
2264
|
-
return true
|
|
2265
|
-
}
|
|
2266
|
-
|
|
2267
|
-
/// 2026-05-21 (v0.3) — decode a JPEG file at the given path into a
|
|
2268
|
-
/// single-channel grayscale CVPixelBuffer (`kCVPixelFormatType_-
|
|
2269
|
-
/// OneComponent8`) suitable for feeding into the C++ KeyframeGate's
|
|
2270
|
-
/// Flow-strategy evaluate path. The bridge's `evaluatePixelBuffer:`
|
|
2271
|
-
/// has explicit OneComponent8 handling (added in v0.3) that reads
|
|
2272
|
-
/// the base address as the Y plane directly, so no extra conversion
|
|
2273
|
-
/// happens on the C++ side.
|
|
2274
|
-
///
|
|
2275
|
-
/// Used by `addBatchKeyframePath` (the JS-driver non-AR path) so the
|
|
2276
|
-
/// Flow strategy actually runs on real pixel data — pre-0.3 this
|
|
2277
|
-
/// path called `evaluate(pose:latchedPlane:)` with no buffer and
|
|
2278
|
-
/// the C++ side silently fell back to Pose strategy.
|
|
2279
|
-
///
|
|
2280
|
-
/// Performance: ~10-20 ms for a 1920×1080 JPEG on iPhone 13/16 Pro.
|
|
2281
|
-
/// Well under the ~250 ms non-AR snapshot interval (~4 FPS).
|
|
2282
|
-
/// v0.4 will replace this path entirely via Frame Processor — see
|
|
2283
|
-
/// issue #11.
|
|
2284
|
-
///
|
|
2285
|
-
/// Returns nil on any failure (file missing, corrupt JPEG, OOM
|
|
2286
|
-
/// on the CVPixelBufferCreate). Callers fall back to the
|
|
2287
|
-
/// pose-only evaluate so the capture doesn't lock up.
|
|
2288
|
-
private static func decodeJpegToGrayscalePixelBuffer(
|
|
2289
|
-
path: String
|
|
2290
|
-
) -> CVPixelBuffer? {
|
|
2291
|
-
let url = URL(fileURLWithPath: path)
|
|
2292
|
-
guard let imageSource = CGImageSourceCreateWithURL(url as CFURL, nil),
|
|
2293
|
-
let cgImage = CGImageSourceCreateImageAtIndex(imageSource, 0, nil)
|
|
2294
|
-
else {
|
|
2295
|
-
return nil
|
|
2296
|
-
}
|
|
2297
|
-
let width = cgImage.width
|
|
2298
|
-
let height = cgImage.height
|
|
2299
|
-
|
|
2300
|
-
var pixelBuffer: CVPixelBuffer?
|
|
2301
|
-
let attrs: NSDictionary = [
|
|
2302
|
-
kCVPixelBufferIOSurfacePropertiesKey: NSDictionary(),
|
|
2303
|
-
]
|
|
2304
|
-
let status = CVPixelBufferCreate(
|
|
2305
|
-
kCFAllocatorDefault,
|
|
2306
|
-
width, height,
|
|
2307
|
-
kCVPixelFormatType_OneComponent8,
|
|
2308
|
-
attrs,
|
|
2309
|
-
&pixelBuffer
|
|
2310
|
-
)
|
|
2311
|
-
guard status == kCVReturnSuccess, let buffer = pixelBuffer else {
|
|
2312
|
-
return nil
|
|
2313
|
-
}
|
|
2314
|
-
|
|
2315
|
-
CVPixelBufferLockBaseAddress(buffer, [])
|
|
2316
|
-
defer { CVPixelBufferUnlockBaseAddress(buffer, []) }
|
|
2317
|
-
guard let baseAddress = CVPixelBufferGetBaseAddress(buffer) else {
|
|
2318
|
-
return nil
|
|
2319
|
-
}
|
|
2320
|
-
let bytesPerRow = CVPixelBufferGetBytesPerRow(buffer)
|
|
2321
|
-
let colorSpace = CGColorSpaceCreateDeviceGray()
|
|
2322
|
-
guard let context = CGContext(
|
|
2323
|
-
data: baseAddress,
|
|
2324
|
-
width: width,
|
|
2325
|
-
height: height,
|
|
2326
|
-
bitsPerComponent: 8,
|
|
2327
|
-
bytesPerRow: bytesPerRow,
|
|
2328
|
-
space: colorSpace,
|
|
2329
|
-
bitmapInfo: CGImageAlphaInfo.none.rawValue
|
|
2330
|
-
) else {
|
|
2331
|
-
return nil
|
|
2332
|
-
}
|
|
2333
|
-
context.draw(cgImage, in: CGRect(x: 0, y: 0, width: width, height: height))
|
|
2334
|
-
return buffer
|
|
2335
|
-
}
|
|
2336
|
-
|
|
2337
2155
|
/// V16 Phase 1 — emit a state event when a batch-keyframe is
|
|
2338
2156
|
/// saved. Carries the on-disk thumbnail path so JS can render it
|
|
2339
2157
|
/// in LiveFrameStrip + advance the "Keyframes: N/M" pill.
|
|
@@ -2342,7 +2160,13 @@ public final class IncrementalStitcher: NSObject {
|
|
|
2342
2160
|
keyframeIndex: Int,
|
|
2343
2161
|
keyframeCount: Int,
|
|
2344
2162
|
keyframeMax: Int,
|
|
2345
|
-
isLandscape: Bool
|
|
2163
|
+
isLandscape: Bool,
|
|
2164
|
+
// v0.7.0 — Tier 1 hook fields. `pose` is the AR pose at the
|
|
2165
|
+
// accept moment (gyro-synthesised in non-AR mode — translation
|
|
2166
|
+
// will read as ~zeros). `acceptedAtMs` is wall-clock ms since
|
|
2167
|
+
// Unix epoch; matches `Date.now()` on the JS side.
|
|
2168
|
+
pose: RNSARFramePose,
|
|
2169
|
+
acceptedAtMs: Double
|
|
2346
2170
|
) {
|
|
2347
2171
|
let state = IncrementalStateObject(
|
|
2348
2172
|
panoramaPath: nil,
|
|
@@ -2366,6 +2190,16 @@ public final class IncrementalStitcher: NSObject {
|
|
|
2366
2190
|
// carry — JS reads these directly from the userInfo blob.
|
|
2367
2191
|
dict["batchKeyframeThumbnailPath"] = thumbnailPath
|
|
2368
2192
|
dict["batchKeyframeIndex"] = keyframeIndex
|
|
2193
|
+
// v0.7.0 — Tier 1 hook (useKeyframeStream) reads these. See
|
|
2194
|
+
// `AcceptedKeyframe` in src/stitching/incremental.ts. Translation
|
|
2195
|
+
// is always emitted; AR mode populates it from the camera
|
|
2196
|
+
// transform, non-AR mode reads ~zeros (gyro-only, no spatial
|
|
2197
|
+
// anchor).
|
|
2198
|
+
dict["batchKeyframePose"] = [
|
|
2199
|
+
"rotation": [pose.qx, pose.qy, pose.qz, pose.qw],
|
|
2200
|
+
"translation": [pose.tx, pose.ty, pose.tz],
|
|
2201
|
+
] as [String: Any]
|
|
2202
|
+
dict["batchKeyframeAcceptedAtMs"] = acceptedAtMs
|
|
2369
2203
|
NotificationCenter.default.post(
|
|
2370
2204
|
name: .retailensIncrementalStateUpdate,
|
|
2371
2205
|
object: nil,
|
|
@@ -2819,7 +2653,12 @@ public final class IncrementalStitcher: NSObject {
|
|
|
2819
2653
|
keyframeIndex: Int(record.index),
|
|
2820
2654
|
keyframeCount: count,
|
|
2821
2655
|
keyframeMax: self.keyframeGate.maxCount,
|
|
2822
|
-
isLandscape: pose.imageWidth >= pose.imageHeight
|
|
2656
|
+
isLandscape: pose.imageWidth >= pose.imageHeight,
|
|
2657
|
+
// v0.7.0 — Tier 1 hook: pose snapshot + accept
|
|
2658
|
+
// timestamp threaded through to JS via the
|
|
2659
|
+
// existing state-update channel.
|
|
2660
|
+
pose: pose,
|
|
2661
|
+
acceptedAtMs: Date().timeIntervalSince1970 * 1000
|
|
2823
2662
|
)
|
|
2824
2663
|
} catch let err as NSError {
|
|
2825
2664
|
os_log(.fault, log: Self.diagLog,
|
|
@@ -3119,14 +2958,15 @@ extension IncrementalStitcher: ARFrameConsumer {}
|
|
|
3119
2958
|
// * `pixelBuffer` from `frame.buffer` (vision-camera YUV biplanar)
|
|
3120
2959
|
// * `tx`/`ty`/`tz` = 0 (no AR translation; gyro only gives rotation)
|
|
3121
2960
|
// * `qx,qy,qz,qw` from JS-thread gyro-integrated yaw+pitch (synthesised
|
|
3122
|
-
// as `q = q_yaw * q_pitch`
|
|
3123
|
-
// `useIncrementalJSDriver`
|
|
2961
|
+
// as `q = q_yaw * q_pitch`). `useFrameProcessorDriver` and (pre-v0.6)
|
|
2962
|
+
// `useIncrementalJSDriver` both produced quaternions with this layout.
|
|
3124
2963
|
// * `fx`/`fy` from frame dims + assumed FoV
|
|
3125
2964
|
// * `cx`/`cy` at image centre
|
|
3126
2965
|
// * `trackingStateRaw = 2` (= `.tracking`) — non-AR captures don't have
|
|
3127
2966
|
// a real ARKit tracking-quality signal; reporting `.tracking` keeps
|
|
3128
|
-
// the engine's `trackingPoor` path inactive
|
|
3129
|
-
// `
|
|
2967
|
+
// the engine's `trackingPoor` path inactive. Both the v0.6+
|
|
2968
|
+
// `useFrameProcessorDriver` and the pre-v0.6 `useIncrementalJSDriver`
|
|
2969
|
+
// follow(ed) this contract.
|
|
3130
2970
|
extension IncrementalStitcher {
|
|
3131
2971
|
@objc public func consumeFrameFromPlugin(
|
|
3132
2972
|
pixelBuffer: CVPixelBuffer,
|
|
@@ -65,14 +65,6 @@ RCT_EXTERN_METHOD(refinePanorama:(NSDictionary *)options
|
|
|
65
65
|
resolver:(RCTPromiseResolveBlock)resolver
|
|
66
66
|
rejecter:(RCTPromiseRejectBlock)rejecter)
|
|
67
67
|
|
|
68
|
-
// 2026-05-17 (Issue #2) — iOS non-AR frame ingestion. JS-side driver
|
|
69
|
-
// hands snapshot file paths + gyro-derived pose to the engine so the
|
|
70
|
-
// live band populates in non-AR mode (parity with Android). See JS
|
|
71
|
-
// `useIncrementalVisionCameraDriver`.
|
|
72
|
-
RCT_EXTERN_METHOD(processFrameAtPath:(NSDictionary *)options
|
|
73
|
-
resolver:(RCTPromiseResolveBlock)resolver
|
|
74
|
-
rejecter:(RCTPromiseRejectBlock)rejecter)
|
|
75
|
-
|
|
76
68
|
// 2026-05-18 (Iss 3) — keyframe storage management. cleanupKeyframes
|
|
77
69
|
// GCs stale per-session directories under Library/Application Support/
|
|
78
70
|
// Captures; getKeyframeDir returns the active capture's session dir.
|