react-native-image-stitcher 0.5.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +200 -8
- package/android/src/main/java/io/imagestitcher/rn/CvFlowGateFrameProcessor.kt +2 -2
- package/android/src/main/java/io/imagestitcher/rn/IncrementalFirstwinsEngine.kt +120 -0
- package/android/src/main/java/io/imagestitcher/rn/IncrementalStitcher.kt +266 -385
- package/android/src/main/java/io/imagestitcher/rn/KeyframeGate.kt +6 -3
- package/android/src/main/java/io/imagestitcher/rn/RNSARCameraView.kt +17 -30
- package/dist/camera/Camera.d.ts +29 -27
- package/dist/camera/Camera.js +48 -79
- package/dist/index.d.ts +0 -2
- package/dist/index.js +4 -6
- package/dist/stitching/incremental.d.ts +10 -11
- package/dist/stitching/useFrameProcessorDriver.d.ts +7 -6
- package/dist/stitching/useFrameProcessorDriver.js +12 -11
- package/ios/Package.swift +35 -21
- package/ios/Sources/RNImageStitcher/IncrementalStitcher.swift +85 -206
- package/ios/Sources/RNImageStitcher/IncrementalStitcherBridge.m +0 -8
- package/ios/Sources/RNImageStitcher/IncrementalStitcherBridge.swift +6 -126
- package/ios/Sources/RNImageStitcher/KeyframeGateFrameProcessor.mm +6 -6
- package/package.json +1 -1
- package/src/camera/Camera.tsx +83 -107
- package/src/index.ts +3 -8
- package/src/stitching/incremental.ts +10 -11
- package/src/stitching/useFrameProcessorDriver.ts +12 -11
- package/dist/stitching/useIncrementalJSDriver.d.ts +0 -74
- package/dist/stitching/useIncrementalJSDriver.js +0 -220
- package/src/stitching/useIncrementalJSDriver.ts +0 -297
|
@@ -365,12 +365,12 @@ public final class IncrementalStitcher: NSObject {
|
|
|
365
365
|
/// F8.3 — gate for `consumeFrameFromPlugin` (the vision-camera
|
|
366
366
|
/// Frame Processor producer-thread entry point). TRUE only when
|
|
367
367
|
/// the current capture was started with
|
|
368
|
-
/// `frameSourceMode == "frameProcessor"`. In
|
|
369
|
-
/// (
|
|
370
|
-
/// `
|
|
371
|
-
///
|
|
372
|
-
///
|
|
373
|
-
///
|
|
368
|
+
/// `frameSourceMode == "frameProcessor"`. In AR mode
|
|
369
|
+
/// (`frameSourceMode == "arSession"`) the plugin would double-feed
|
|
370
|
+
/// the engine alongside ARKit's `consumeFrame` delegate path —
|
|
371
|
+
/// pixel buffers from the producer thread + pixel buffers from the
|
|
372
|
+
/// ARSession delegate, racing on the same workQueue — so we drop
|
|
373
|
+
/// the producer-thread call.
|
|
374
374
|
///
|
|
375
375
|
/// Set under `stateLock` in `start()`, cleared under `stateLock`
|
|
376
376
|
/// in `cancel()` and `finalize()`, ALSO read under `stateLock`
|
|
@@ -476,6 +476,13 @@ public final class IncrementalStitcher: NSObject {
|
|
|
476
476
|
|
|
477
477
|
private override init() {
|
|
478
478
|
super.init()
|
|
479
|
+
// F8.3.H2 — runtime check that Swift's auto-bridged ObjC
|
|
480
|
+
// selector for `consumeFrameFromPlugin(...)` matches the
|
|
481
|
+
// selector string the plugin's .mm dispatches. Asserts in
|
|
482
|
+
// dev builds; no-ops in release. See the
|
|
483
|
+
// `_consumeFrameFromPluginSelectorPin` declaration below for
|
|
484
|
+
// the full rationale.
|
|
485
|
+
IncrementalStitcher._verifyConsumeFrameFromPluginSelector()
|
|
479
486
|
}
|
|
480
487
|
|
|
481
488
|
/// 2026-05-18 (iOS cross-orientation fix) — bridge entry-point
|
|
@@ -740,11 +747,13 @@ public final class IncrementalStitcher: NSObject {
|
|
|
740
747
|
engineMode: String,
|
|
741
748
|
captureOrientation: String = "portrait",
|
|
742
749
|
configOverrides: [String: Any] = [:],
|
|
743
|
-
// 2026-05-18 (Issue #2 regression fix): "arSession" (default
|
|
744
|
-
//
|
|
745
|
-
// "
|
|
746
|
-
// via
|
|
747
|
-
//
|
|
750
|
+
// 2026-05-18 (Issue #2 regression fix): "arSession" (default)
|
|
751
|
+
// registers as the ARSession's frame consumer.
|
|
752
|
+
// "frameProcessor" skips that registration — frames come in
|
|
753
|
+
// via the vision-camera Frame Processor plugin's
|
|
754
|
+
// `consumeFrameFromPlugin` path instead. The pre-v0.6
|
|
755
|
+
// "jsDriver" mode (push frames in from JS via
|
|
756
|
+
// processFrameAtPath) has been removed.
|
|
748
757
|
frameSourceMode: String = "arSession"
|
|
749
758
|
) {
|
|
750
759
|
stateLock.lock()
|
|
@@ -940,9 +949,9 @@ public final class IncrementalStitcher: NSObject {
|
|
|
940
949
|
}
|
|
941
950
|
self.isRunning = true
|
|
942
951
|
// F8.3 — enable the Frame Processor plugin's producer-thread
|
|
943
|
-
// ingest only for the new "frameProcessor" mode.
|
|
944
|
-
//
|
|
945
|
-
//
|
|
952
|
+
// ingest only for the new "frameProcessor" mode. AR mode
|
|
953
|
+
// ("arSession") keeps it OFF; see the ivar's declaration
|
|
954
|
+
// comment for why.
|
|
946
955
|
self.frameProcessorIngestEnabled = (frameSourceMode == "frameProcessor")
|
|
947
956
|
self.snapshotJpegQuality = max(1, min(100, snapshotJpegQuality))
|
|
948
957
|
self.snapshotEveryNAccepts = max(1, snapshotEveryNAccepts)
|
|
@@ -1092,9 +1101,10 @@ public final class IncrementalStitcher: NSObject {
|
|
|
1092
1101
|
// `stateLock.try()` and corrupting
|
|
1093
1102
|
// the gate's novelty math.
|
|
1094
1103
|
// (Adversarial-review C1.)
|
|
1095
|
-
//
|
|
1096
|
-
//
|
|
1097
|
-
//
|
|
1104
|
+
//
|
|
1105
|
+
// The pre-v0.6 `jsDriver` mode (which pushed frames via
|
|
1106
|
+
// `processFrameAtPath` and also skipped registration) has
|
|
1107
|
+
// been removed.
|
|
1098
1108
|
if frameSourceMode == "arSession" {
|
|
1099
1109
|
RNSARSession.shared.incrementalConsumer = self
|
|
1100
1110
|
}
|
|
@@ -2142,191 +2152,6 @@ public final class IncrementalStitcher: NSObject {
|
|
|
2142
2152
|
self.keyframeGate.acceptedCount, self.keyframeGate.maxCount)
|
|
2143
2153
|
}
|
|
2144
2154
|
|
|
2145
|
-
/// Whether the engine is currently in batch-keyframe mode.
|
|
2146
|
-
/// Bridge reads this to decide whether the JS-driven
|
|
2147
|
-
/// `processFrameAtPath` path can use the lightweight
|
|
2148
|
-
/// `addBatchKeyframePath` (path-only) entry below.
|
|
2149
|
-
@objc public var isBatchKeyframeMode: Bool {
|
|
2150
|
-
stateLock.lock()
|
|
2151
|
-
defer { stateLock.unlock() }
|
|
2152
|
-
return batchKeyframeMode
|
|
2153
|
-
}
|
|
2154
|
-
|
|
2155
|
-
/// 2026-05-18 (Issue #2 v2) — JS-driver entry-point for
|
|
2156
|
-
/// batch-keyframe captures. Mirrors Android's behaviour: the
|
|
2157
|
-
/// caller (JS side via the IncrementalStitcherBridge) hands us a
|
|
2158
|
-
/// JPEG file path that already exists on disk (saved by
|
|
2159
|
-
/// vision-camera's takeSnapshot), plus a synthetic pose derived
|
|
2160
|
-
/// from gyro integration. We:
|
|
2161
|
-
///
|
|
2162
|
-
/// 1. Validate state (running + batchKeyframeMode).
|
|
2163
|
-
/// 2. Ask the shared C++ KeyframeGate whether to accept this
|
|
2164
|
-
/// frame. Pass `latchedPlane: nil` — non-AR captures have
|
|
2165
|
-
/// no plane; the C++ gate falls back to a pose-only
|
|
2166
|
-
/// angular-delta strategy. We do NOT pass a pixel buffer:
|
|
2167
|
-
/// the Pose strategy doesn't need one, and avoiding the
|
|
2168
|
-
/// JPEG → CVPixelBuffer round-trip dodges the iOS
|
|
2169
|
-
/// orientation bugs that broke Issue 2 v1
|
|
2170
|
-
/// (UIImage/CGContext Y-flip + EXIF-vs-CGImage dimension
|
|
2171
|
-
/// mismatch — see the symptom in 2026-05-18 user report).
|
|
2172
|
-
/// 3. If accepted, append the existing path + pose to the
|
|
2173
|
-
/// finalize-time lists. No JPEG re-encode — the file on
|
|
2174
|
-
/// disk IS the keyframe. `retailens::stitchFramePaths()`
|
|
2175
|
-
/// at finalize uses `cv::imread` which natively handles
|
|
2176
|
-
/// EXIF orientation, so the output panorama reads upright.
|
|
2177
|
-
/// 4. Emit the same state-event the AR delegate path emits so
|
|
2178
|
-
/// the JS live band populates identically.
|
|
2179
|
-
///
|
|
2180
|
-
/// Architecture note: this is structurally parallel to Android's
|
|
2181
|
-
/// `IncrementalStitcher.kt::processFrameAtPath`
|
|
2182
|
-
/// `batchKeyframeMode` branch (lines 573-627). A follow-up
|
|
2183
|
-
/// should extract the dispatch (gate-eval + path-append + emit)
|
|
2184
|
-
/// into shared cpp/ so both platforms become 5-line wrappers
|
|
2185
|
-
/// around a single C++ entry point.
|
|
2186
|
-
@objc public func addBatchKeyframePath(
|
|
2187
|
-
path: String,
|
|
2188
|
-
pose: RNSARFramePose
|
|
2189
|
-
) -> Bool {
|
|
2190
|
-
stateLock.lock()
|
|
2191
|
-
guard self.isRunning, self.batchKeyframeMode else {
|
|
2192
|
-
stateLock.unlock()
|
|
2193
|
-
return false
|
|
2194
|
-
}
|
|
2195
|
-
stateLock.unlock()
|
|
2196
|
-
|
|
2197
|
-
// 2026-05-21 (v0.3) — pixel-aware Flow-strategy evaluation.
|
|
2198
|
-
// Pre-0.3 this was `evaluate(pose:latchedPlane:)` with no pixel
|
|
2199
|
-
// buffer, which forced the C++ gate to silently fall back to
|
|
2200
|
-
// Pose strategy (same bug as Android non-AR; both fixed in
|
|
2201
|
-
// v0.3). We now decode the JPEG snapshot at `path` to a
|
|
2202
|
-
// single-channel grayscale CVPixelBuffer and pass it through,
|
|
2203
|
-
// so the gate's Flow strategy actually runs sparse-flow
|
|
2204
|
-
// novelty on real image content.
|
|
2205
|
-
//
|
|
2206
|
-
// CGImageSource → CGContext into a OneComponent8 CVPixelBuffer.
|
|
2207
|
-
// ~10-20 ms per snapshot on iPhone 13/16 Pro; well under the
|
|
2208
|
-
// ~250 ms non-AR snapshot interval (~4 FPS cadence). v0.4
|
|
2209
|
-
// will replace this path entirely by moving non-AR capture to
|
|
2210
|
-
// vision-camera's Frame Processor API (tracked at issue #11).
|
|
2211
|
-
let decision: KeyframeGateDecision
|
|
2212
|
-
if let grayBuffer = Self.decodeJpegToGrayscalePixelBuffer(path: path) {
|
|
2213
|
-
decision = self.keyframeGate.evaluate(
|
|
2214
|
-
pose: pose,
|
|
2215
|
-
latchedPlane: nil,
|
|
2216
|
-
pixelBuffer: grayBuffer
|
|
2217
|
-
)
|
|
2218
|
-
} else {
|
|
2219
|
-
// JPEG decode failed (corrupt file, OOM, etc.). Fall back
|
|
2220
|
-
// to pose-only so the capture doesn't lock up — matches
|
|
2221
|
-
// the C++ side's defensive grayData==nullptr handling
|
|
2222
|
-
// inside evaluateWithFrame.
|
|
2223
|
-
decision = self.keyframeGate.evaluate(
|
|
2224
|
-
pose: pose,
|
|
2225
|
-
latchedPlane: nil
|
|
2226
|
-
)
|
|
2227
|
-
}
|
|
2228
|
-
if !decision.accept {
|
|
2229
|
-
self.emitKeyframeRejectState(decision: decision)
|
|
2230
|
-
return false
|
|
2231
|
-
}
|
|
2232
|
-
|
|
2233
|
-
// Append path + pose to the finalize lists. Take the lock
|
|
2234
|
-
// briefly — these mutate state read by `finalize()`.
|
|
2235
|
-
stateLock.lock()
|
|
2236
|
-
self.keyframePaths.append(path)
|
|
2237
|
-
self.keyframePoses.append(pose.asDictionary())
|
|
2238
|
-
// 2026-05-22 (audit F2) — track first + last pose for the
|
|
2239
|
-
// stitchMode auto-resolver. iOS parity: Android records
|
|
2240
|
-
// these in IncrementalStitcher.kt at the same accept points.
|
|
2241
|
-
let poseArr = [pose.tx, pose.ty, pose.tz,
|
|
2242
|
-
pose.qx, pose.qy, pose.qz, pose.qw]
|
|
2243
|
-
if self.batchFirstAcceptedPose == nil { self.batchFirstAcceptedPose = poseArr }
|
|
2244
|
-
self.batchLastAcceptedPose = poseArr
|
|
2245
|
-
let count = self.keyframePaths.count
|
|
2246
|
-
stateLock.unlock()
|
|
2247
|
-
os_log(.fault, log: Self.diagLog,
|
|
2248
|
-
"[V16-batch-keyframe.js] accepted path #%d → %{public}@",
|
|
2249
|
-
Int32(count), path)
|
|
2250
|
-
self.emitBatchKeyframeAcceptedState(
|
|
2251
|
-
thumbnailPath: path,
|
|
2252
|
-
keyframeIndex: count - 1,
|
|
2253
|
-
keyframeCount: count,
|
|
2254
|
-
keyframeMax: self.keyframeGate.maxCount,
|
|
2255
|
-
isLandscape: pose.imageWidth >= pose.imageHeight
|
|
2256
|
-
)
|
|
2257
|
-
return true
|
|
2258
|
-
}
|
|
2259
|
-
|
|
2260
|
-
/// 2026-05-21 (v0.3) — decode a JPEG file at the given path into a
|
|
2261
|
-
/// single-channel grayscale CVPixelBuffer (`kCVPixelFormatType_-
|
|
2262
|
-
/// OneComponent8`) suitable for feeding into the C++ KeyframeGate's
|
|
2263
|
-
/// Flow-strategy evaluate path. The bridge's `evaluatePixelBuffer:`
|
|
2264
|
-
/// has explicit OneComponent8 handling (added in v0.3) that reads
|
|
2265
|
-
/// the base address as the Y plane directly, so no extra conversion
|
|
2266
|
-
/// happens on the C++ side.
|
|
2267
|
-
///
|
|
2268
|
-
/// Used by `addBatchKeyframePath` (the JS-driver non-AR path) so the
|
|
2269
|
-
/// Flow strategy actually runs on real pixel data — pre-0.3 this
|
|
2270
|
-
/// path called `evaluate(pose:latchedPlane:)` with no buffer and
|
|
2271
|
-
/// the C++ side silently fell back to Pose strategy.
|
|
2272
|
-
///
|
|
2273
|
-
/// Performance: ~10-20 ms for a 1920×1080 JPEG on iPhone 13/16 Pro.
|
|
2274
|
-
/// Well under the ~250 ms non-AR snapshot interval (~4 FPS).
|
|
2275
|
-
/// v0.4 will replace this path entirely via Frame Processor — see
|
|
2276
|
-
/// issue #11.
|
|
2277
|
-
///
|
|
2278
|
-
/// Returns nil on any failure (file missing, corrupt JPEG, OOM
|
|
2279
|
-
/// on the CVPixelBufferCreate). Callers fall back to the
|
|
2280
|
-
/// pose-only evaluate so the capture doesn't lock up.
|
|
2281
|
-
private static func decodeJpegToGrayscalePixelBuffer(
|
|
2282
|
-
path: String
|
|
2283
|
-
) -> CVPixelBuffer? {
|
|
2284
|
-
let url = URL(fileURLWithPath: path)
|
|
2285
|
-
guard let imageSource = CGImageSourceCreateWithURL(url as CFURL, nil),
|
|
2286
|
-
let cgImage = CGImageSourceCreateImageAtIndex(imageSource, 0, nil)
|
|
2287
|
-
else {
|
|
2288
|
-
return nil
|
|
2289
|
-
}
|
|
2290
|
-
let width = cgImage.width
|
|
2291
|
-
let height = cgImage.height
|
|
2292
|
-
|
|
2293
|
-
var pixelBuffer: CVPixelBuffer?
|
|
2294
|
-
let attrs: NSDictionary = [
|
|
2295
|
-
kCVPixelBufferIOSurfacePropertiesKey: NSDictionary(),
|
|
2296
|
-
]
|
|
2297
|
-
let status = CVPixelBufferCreate(
|
|
2298
|
-
kCFAllocatorDefault,
|
|
2299
|
-
width, height,
|
|
2300
|
-
kCVPixelFormatType_OneComponent8,
|
|
2301
|
-
attrs,
|
|
2302
|
-
&pixelBuffer
|
|
2303
|
-
)
|
|
2304
|
-
guard status == kCVReturnSuccess, let buffer = pixelBuffer else {
|
|
2305
|
-
return nil
|
|
2306
|
-
}
|
|
2307
|
-
|
|
2308
|
-
CVPixelBufferLockBaseAddress(buffer, [])
|
|
2309
|
-
defer { CVPixelBufferUnlockBaseAddress(buffer, []) }
|
|
2310
|
-
guard let baseAddress = CVPixelBufferGetBaseAddress(buffer) else {
|
|
2311
|
-
return nil
|
|
2312
|
-
}
|
|
2313
|
-
let bytesPerRow = CVPixelBufferGetBytesPerRow(buffer)
|
|
2314
|
-
let colorSpace = CGColorSpaceCreateDeviceGray()
|
|
2315
|
-
guard let context = CGContext(
|
|
2316
|
-
data: baseAddress,
|
|
2317
|
-
width: width,
|
|
2318
|
-
height: height,
|
|
2319
|
-
bitsPerComponent: 8,
|
|
2320
|
-
bytesPerRow: bytesPerRow,
|
|
2321
|
-
space: colorSpace,
|
|
2322
|
-
bitmapInfo: CGImageAlphaInfo.none.rawValue
|
|
2323
|
-
) else {
|
|
2324
|
-
return nil
|
|
2325
|
-
}
|
|
2326
|
-
context.draw(cgImage, in: CGRect(x: 0, y: 0, width: width, height: height))
|
|
2327
|
-
return buffer
|
|
2328
|
-
}
|
|
2329
|
-
|
|
2330
2155
|
/// V16 Phase 1 — emit a state event when a batch-keyframe is
|
|
2331
2156
|
/// saved. Carries the on-disk thumbnail path so JS can render it
|
|
2332
2157
|
/// in LiveFrameStrip + advance the "Keyframes: N/M" pill.
|
|
@@ -3112,14 +2937,15 @@ extension IncrementalStitcher: ARFrameConsumer {}
|
|
|
3112
2937
|
// * `pixelBuffer` from `frame.buffer` (vision-camera YUV biplanar)
|
|
3113
2938
|
// * `tx`/`ty`/`tz` = 0 (no AR translation; gyro only gives rotation)
|
|
3114
2939
|
// * `qx,qy,qz,qw` from JS-thread gyro-integrated yaw+pitch (synthesised
|
|
3115
|
-
// as `q = q_yaw * q_pitch`
|
|
3116
|
-
// `useIncrementalJSDriver`
|
|
2940
|
+
// as `q = q_yaw * q_pitch`). `useFrameProcessorDriver` and (pre-v0.6)
|
|
2941
|
+
// `useIncrementalJSDriver` both produced quaternions with this layout.
|
|
3117
2942
|
// * `fx`/`fy` from frame dims + assumed FoV
|
|
3118
2943
|
// * `cx`/`cy` at image centre
|
|
3119
2944
|
// * `trackingStateRaw = 2` (= `.tracking`) — non-AR captures don't have
|
|
3120
2945
|
// a real ARKit tracking-quality signal; reporting `.tracking` keeps
|
|
3121
|
-
// the engine's `trackingPoor` path inactive
|
|
3122
|
-
// `
|
|
2946
|
+
// the engine's `trackingPoor` path inactive. Both the v0.6+
|
|
2947
|
+
// `useFrameProcessorDriver` and the pre-v0.6 `useIncrementalJSDriver`
|
|
2948
|
+
// follow(ed) this contract.
|
|
3123
2949
|
extension IncrementalStitcher {
|
|
3124
2950
|
@objc public func consumeFrameFromPlugin(
|
|
3125
2951
|
pixelBuffer: CVPixelBuffer,
|
|
@@ -3158,4 +2984,57 @@ extension IncrementalStitcher {
|
|
|
3158
2984
|
)
|
|
3159
2985
|
consumeFrame(pixelBuffer: pixelBuffer, pose: pose)
|
|
3160
2986
|
}
|
|
2987
|
+
|
|
2988
|
+
// F8.3.H2 — compile-time + runtime guard for the Swift⇄ObjC
|
|
2989
|
+
// selector contract that `KeyframeGateFrameProcessor.mm`
|
|
2990
|
+
// depends on.
|
|
2991
|
+
//
|
|
2992
|
+
// The .mm file forward-declares `IncrementalStitcher` and
|
|
2993
|
+
// dispatches `[shared consumeFrameFromPluginWithPixelBuffer:tx:
|
|
2994
|
+
// …:trackingStateRaw:]` by NAME — ObjC's late-binding means
|
|
2995
|
+
// signature drift would silently link but crash at runtime
|
|
2996
|
+
// with `NSInvalidArgumentException: unrecognized selector`
|
|
2997
|
+
// on the first non-AR frame.
|
|
2998
|
+
//
|
|
2999
|
+
// This `#selector(...)` reference forces the Swift compiler
|
|
3000
|
+
// to resolve the exact method signature. If anyone renames a
|
|
3001
|
+
// parameter label or adds/removes an argument, the
|
|
3002
|
+
// `_consumeFrameFromPluginSelectorPin` expression fails to
|
|
3003
|
+
// compile — the SDK won't build until the .mm's forward
|
|
3004
|
+
// declaration is updated to match. Stronger guarantee than a
|
|
3005
|
+
// test that needs iOS-Simulator infrastructure to run.
|
|
3006
|
+
//
|
|
3007
|
+
// The runtime check below additionally pins the exact
|
|
3008
|
+
// SELECTOR STRING the .mm dispatches. In dev/debug builds it
|
|
3009
|
+
// asserts; in release builds it's a no-op (the static let is
|
|
3010
|
+
// initialised lazily and never read otherwise, so the runtime
|
|
3011
|
+
// cost is one-time + tiny). Drift between Swift's auto-
|
|
3012
|
+
// generated selector name and the .mm's expected string
|
|
3013
|
+
// (e.g., if Swift's bridging rules change) trips the assert.
|
|
3014
|
+
private static let _consumeFrameFromPluginSelectorPin: Selector =
|
|
3015
|
+
#selector(IncrementalStitcher.consumeFrameFromPlugin(
|
|
3016
|
+
pixelBuffer:
|
|
3017
|
+
tx: ty: tz:
|
|
3018
|
+
qx: qy: qz: qw:
|
|
3019
|
+
fx: fy: cx: cy:
|
|
3020
|
+
imageWidth: imageHeight:
|
|
3021
|
+
timestampMs:
|
|
3022
|
+
trackingStateRaw:))
|
|
3023
|
+
|
|
3024
|
+
@inline(never)
|
|
3025
|
+
private static func _verifyConsumeFrameFromPluginSelector() {
|
|
3026
|
+
let expected =
|
|
3027
|
+
"consumeFrameFromPluginWithPixelBuffer:tx:ty:tz:"
|
|
3028
|
+
+ "qx:qy:qz:qw:fx:fy:cx:cy:"
|
|
3029
|
+
+ "imageWidth:imageHeight:timestampMs:trackingStateRaw:"
|
|
3030
|
+
let actual = NSStringFromSelector(_consumeFrameFromPluginSelectorPin)
|
|
3031
|
+
assert(
|
|
3032
|
+
actual == expected,
|
|
3033
|
+
"Frame Processor selector drift — Swift's auto-bridged "
|
|
3034
|
+
+ "ObjC selector for consumeFrameFromPlugin is "
|
|
3035
|
+
+ "\(actual) but KeyframeGateFrameProcessor.mm's "
|
|
3036
|
+
+ "forward declaration expects \(expected). Update the "
|
|
3037
|
+
+ ".mm to match (or fix the assumption here).",
|
|
3038
|
+
)
|
|
3039
|
+
}
|
|
3161
3040
|
}
|
|
@@ -65,14 +65,6 @@ RCT_EXTERN_METHOD(refinePanorama:(NSDictionary *)options
|
|
|
65
65
|
resolver:(RCTPromiseResolveBlock)resolver
|
|
66
66
|
rejecter:(RCTPromiseRejectBlock)rejecter)
|
|
67
67
|
|
|
68
|
-
// 2026-05-17 (Issue #2) — iOS non-AR frame ingestion. JS-side driver
|
|
69
|
-
// hands snapshot file paths + gyro-derived pose to the engine so the
|
|
70
|
-
// live band populates in non-AR mode (parity with Android). See JS
|
|
71
|
-
// `useIncrementalVisionCameraDriver`.
|
|
72
|
-
RCT_EXTERN_METHOD(processFrameAtPath:(NSDictionary *)options
|
|
73
|
-
resolver:(RCTPromiseResolveBlock)resolver
|
|
74
|
-
rejecter:(RCTPromiseRejectBlock)rejecter)
|
|
75
|
-
|
|
76
68
|
// 2026-05-18 (Iss 3) — keyframe storage management. cleanupKeyframes
|
|
77
69
|
// GCs stale per-session directories under Library/Application Support/
|
|
78
70
|
// Captures; getKeyframeDir returns the active capture's session dir.
|
|
@@ -20,7 +20,6 @@
|
|
|
20
20
|
import Foundation
|
|
21
21
|
import React
|
|
22
22
|
import os.log
|
|
23
|
-
import ImageIO // CGImageSource + kCGImagePropertyOrientation for EXIF read in processFrameAtPath
|
|
24
23
|
|
|
25
24
|
@objc(IncrementalStitcherBridge)
|
|
26
25
|
public final class IncrementalStitcherBridge: RCTEventEmitter {
|
|
@@ -75,10 +74,12 @@ public final class IncrementalStitcherBridge: RCTEventEmitter {
|
|
|
75
74
|
/// Resolves with `{ ok: true }`. Rejects when `frameSourceMode`
|
|
76
75
|
/// (options dict) is 'arSession' (the default) AND the AR session
|
|
77
76
|
/// isn't running — that path needs ARKit to deliver frames.
|
|
78
|
-
/// When `frameSourceMode` is '
|
|
79
|
-
/// skipped and the engine expects
|
|
80
|
-
/// `
|
|
81
|
-
///
|
|
77
|
+
/// When `frameSourceMode` is 'frameProcessor' the AR-session check
|
|
78
|
+
/// is skipped and the engine expects the vision-camera Frame
|
|
79
|
+
/// Processor plugin (`CvFlowGateFrameProcessor`) to feed frames
|
|
80
|
+
/// via `consumeFrameFromPlugin`. The pre-v0.6 'jsDriver' mode
|
|
81
|
+
/// (push frames in from JS via `processFrameAtPath`) has been
|
|
82
|
+
/// removed.
|
|
82
83
|
@objc(start:resolver:rejecter:)
|
|
83
84
|
public func start(
|
|
84
85
|
options: NSDictionary,
|
|
@@ -251,127 +252,6 @@ public final class IncrementalStitcherBridge: RCTEventEmitter {
|
|
|
251
252
|
resolver(["ok": true])
|
|
252
253
|
}
|
|
253
254
|
|
|
254
|
-
/// 2026-05-18 (Issue #2 v2) — JS-driven frame ingestion for iOS
|
|
255
|
-
/// non-AR mode. Mirrors Android's `processFrameAtPath` exactly:
|
|
256
|
-
/// the JPEG at `path` is already saved on disk by vision-camera
|
|
257
|
-
/// in its native EXIF-correct orientation. We DO NOT decode the
|
|
258
|
-
/// image here. Instead:
|
|
259
|
-
///
|
|
260
|
-
/// - Build a synthetic `RNSARFramePose` from the
|
|
261
|
-
/// JS-supplied quaternion + intrinsics (no translation;
|
|
262
|
-
/// non-AR captures don't have it).
|
|
263
|
-
/// - Hand the path + pose to
|
|
264
|
-
/// `IncrementalStitcher.addBatchKeyframePath`, which
|
|
265
|
-
/// evaluates the shared-C++ KeyframeGate and (if accepted)
|
|
266
|
-
/// records the path in the finalize-time keyframe list +
|
|
267
|
-
/// emits the same state event the AR-delegate path emits.
|
|
268
|
-
/// - `cv::imread` at finalize handles EXIF orientation
|
|
269
|
-
/// natively, so the output panorama reads upright with no
|
|
270
|
-
/// iOS-specific orientation handling needed in this bridge.
|
|
271
|
-
///
|
|
272
|
-
/// History: Issue #2 v1 (commit 0e40f17) tried to decode the
|
|
273
|
-
/// JPEG into a CVPixelBuffer and reuse the existing AR
|
|
274
|
-
/// `consumeFrame(pixelBuffer:pose:)` path. That introduced two
|
|
275
|
-
/// orientation bugs (CGContext Y-flip + UIImage.size vs
|
|
276
|
-
/// cgImage.width dim swap) → upside-down output AND canvas-
|
|
277
|
-
/// dimension overflow → OOM crashes (user-reported 2026-05-18).
|
|
278
|
-
/// Architecturally Android never decoded the image either, so
|
|
279
|
-
/// the right fix was to mirror that.
|
|
280
|
-
///
|
|
281
|
-
/// `options` keys:
|
|
282
|
-
/// - path (NSString, required) — local file path (no file://)
|
|
283
|
-
/// - qx, qy, qz, qw (Double, required) — quaternion, JS-side
|
|
284
|
-
/// gyro-integrated
|
|
285
|
-
/// - fx, fy, cx, cy (Double, required) — intrinsics in sensor px
|
|
286
|
-
/// - imageWidth, imageHeight (Int, required)
|
|
287
|
-
/// - trackingPoor (Bool, optional, default false)
|
|
288
|
-
/// - timestampMs (Double, optional, default = now)
|
|
289
|
-
///
|
|
290
|
-
/// Only batch-keyframe captures are supported on this path right
|
|
291
|
-
/// now — other engines (hybrid / firstwins) need real pixel data
|
|
292
|
-
/// during the live phase, which isn't trivially derivable from a
|
|
293
|
-
/// JPEG path. Reject with `E_NOT_BATCH_KEYFRAME` so the JS host
|
|
294
|
-
/// can fall back to the legacy stitchVideo path if needed.
|
|
295
|
-
@objc(processFrameAtPath:resolver:rejecter:)
|
|
296
|
-
public func processFrameAtPath(
|
|
297
|
-
options: NSDictionary,
|
|
298
|
-
resolver: @escaping RCTPromiseResolveBlock,
|
|
299
|
-
rejecter: @escaping RCTPromiseRejectBlock
|
|
300
|
-
) {
|
|
301
|
-
guard let pathRaw = options["path"] as? String, !pathRaw.isEmpty else {
|
|
302
|
-
rejecter("E_NO_PATH", "processFrameAtPath: missing 'path'", nil)
|
|
303
|
-
return
|
|
304
|
-
}
|
|
305
|
-
// Strip optional file:// prefix — JS callers sometimes send
|
|
306
|
-
// file URIs, native APIs want filesystem paths.
|
|
307
|
-
let cleanPath = pathRaw.hasPrefix("file://")
|
|
308
|
-
? String(pathRaw.dropFirst("file://".count))
|
|
309
|
-
: pathRaw
|
|
310
|
-
|
|
311
|
-
let engine = IncrementalStitcher.shared
|
|
312
|
-
guard engine.isBatchKeyframeMode else {
|
|
313
|
-
rejecter("E_NOT_BATCH_KEYFRAME",
|
|
314
|
-
"processFrameAtPath only supports batch-keyframe "
|
|
315
|
-
+ "engine mode on iOS. Configure "
|
|
316
|
-
+ "incrementalEngine='batch-keyframe' in start() "
|
|
317
|
-
+ "options, or fall back to the stitchVideo path.",
|
|
318
|
-
nil)
|
|
319
|
-
return
|
|
320
|
-
}
|
|
321
|
-
|
|
322
|
-
let qx = (options["qx"] as? Double) ?? 0
|
|
323
|
-
let qy = (options["qy"] as? Double) ?? 0
|
|
324
|
-
let qz = (options["qz"] as? Double) ?? 0
|
|
325
|
-
let qw = (options["qw"] as? Double) ?? 1 // identity quat default
|
|
326
|
-
let fx = (options["fx"] as? Double) ?? 1000.0
|
|
327
|
-
let fy = (options["fy"] as? Double) ?? 1000.0
|
|
328
|
-
let cx = (options["cx"] as? Double) ?? 540.0
|
|
329
|
-
let cy = (options["cy"] as? Double) ?? 960.0
|
|
330
|
-
let imageWidth = (options["imageWidth"] as? Int) ?? 1080
|
|
331
|
-
let imageHeight = (options["imageHeight"] as? Int) ?? 1920
|
|
332
|
-
let trackingPoor = (options["trackingPoor"] as? Bool) ?? false
|
|
333
|
-
let timestampMs = (options["timestampMs"] as? Double)
|
|
334
|
-
?? (Date().timeIntervalSince1970 * 1000.0)
|
|
335
|
-
let trackingState: RNSARTrackingState =
|
|
336
|
-
trackingPoor ? .limited : .tracking
|
|
337
|
-
|
|
338
|
-
let pose = RNSARFramePose(
|
|
339
|
-
tx: 0, ty: 0, tz: 0, // no translation in non-AR
|
|
340
|
-
qx: qx, qy: qy, qz: qz, qw: qw,
|
|
341
|
-
fx: fx, fy: fy, cx: cx, cy: cy,
|
|
342
|
-
imageWidth: imageWidth, imageHeight: imageHeight,
|
|
343
|
-
timestampMs: timestampMs,
|
|
344
|
-
trackingState: trackingState
|
|
345
|
-
)
|
|
346
|
-
|
|
347
|
-
// 2026-05-18 (Iss #1 diag) — read EXIF Orientation tag from the
|
|
348
|
-
// keyframe JPEG before handing it to the engine. vision-camera
|
|
349
|
-
// writes a JPEG with an EXIF tag matching the physical capture
|
|
350
|
-
// orientation (1=no rotation, 3=180°, 6=90°CW, 8=90°CCW). The
|
|
351
|
-
// bake-rotation table in cpp/stitcher.cpp assumes the post-imread
|
|
352
|
-
// Mat is in user-view orientation (post-EXIF apply). If the EXIF
|
|
353
|
-
// tag isn't what we expect for a given physical orientation, the
|
|
354
|
-
// input Mat to cv::Stitcher will be a different shape than the AR
|
|
355
|
-
// path produces (AR keyframes hardcode EXIF=6, commit 7b828f1) —
|
|
356
|
-
// which would explain why iOS non-AR landscape captures stitch
|
|
357
|
-
// but bake the wrong way. CGImageSource is cheap (metadata-only;
|
|
358
|
-
// no decode).
|
|
359
|
-
var exifOrientation: Int = -1
|
|
360
|
-
if let src = CGImageSourceCreateWithURL(
|
|
361
|
-
URL(fileURLWithPath: cleanPath) as CFURL, nil
|
|
362
|
-
),
|
|
363
|
-
let props = CGImageSourceCopyPropertiesAtIndex(src, 0, nil) as? [CFString: Any],
|
|
364
|
-
let o = props[kCGImagePropertyOrientation] as? Int {
|
|
365
|
-
exifOrientation = o
|
|
366
|
-
}
|
|
367
|
-
os_log(.fault, log: OSLog(subsystem: "com.tiger.retailens",
|
|
368
|
-
category: "stitcher.diag"),
|
|
369
|
-
"[V16-batch-keyframe.js] processFrameAtPath EXIF=%d imageW=%d imageH=%d path=%{public}@",
|
|
370
|
-
Int32(exifOrientation), Int32(imageWidth), Int32(imageHeight), cleanPath)
|
|
371
|
-
|
|
372
|
-
let accepted = engine.addBatchKeyframePath(path: cleanPath, pose: pose)
|
|
373
|
-
resolver(["ok": true, "accepted": accepted])
|
|
374
|
-
}
|
|
375
255
|
|
|
376
256
|
/// 2026-05-18 (Iss 3) — bridge for `cleanupKeyframes`. See the
|
|
377
257
|
/// Swift method's docstring for behaviour. Options dict keys:
|
|
@@ -140,13 +140,13 @@ static NSInteger kg_argInt(NSDictionary* args, NSString* key, NSInteger defaultV
|
|
|
140
140
|
// Pose from worklet args. Defaults are safe non-AR values:
|
|
141
141
|
// * tx/ty/tz = 0 (no translation in non-AR; gyro only gives rot)
|
|
142
142
|
// * qw = 1 (identity quaternion if JS hasn't supplied rotation)
|
|
143
|
-
// * fx/fy/cx/cy = 0 →
|
|
144
|
-
// engine derives FoV from intrinsics; 0 would
|
|
145
|
-
// We default the principal point to image
|
|
146
|
-
// fallback if only fx/fy are missing.
|
|
143
|
+
// * fx/fy/cx/cy = 0 → the Frame Processor worklet caller MUST
|
|
144
|
+
// supply these (the engine derives FoV from intrinsics; 0 would
|
|
145
|
+
// yield NaN FoV). We default the principal point to image
|
|
146
|
+
// centre as a safer fallback if only fx/fy are missing.
|
|
147
147
|
// * trackingStateRaw = 2 → `.tracking` (non-AR captures don't
|
|
148
|
-
// have a real tracking-quality signal;
|
|
149
|
-
//
|
|
148
|
+
// have a real tracking-quality signal; reporting `.tracking`
|
|
149
|
+
// keeps the engine's `trackingPoor` path inactive).
|
|
150
150
|
double tx = kg_argDouble(arguments, @"tx", 0.0);
|
|
151
151
|
double ty = kg_argDouble(arguments, @"ty", 0.0);
|
|
152
152
|
double tz = kg_argDouble(arguments, @"tz", 0.0);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "react-native-image-stitcher",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.6.0",
|
|
4
4
|
"description": "Pose-aware panorama capture + stitching for React Native. One <Camera> component, both tap-to-photo and hold-to-pan modes, both AR-backed and IMU-fallback capture paths.",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|