react-native-image-stitcher 0.14.1 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +160 -0
- package/README.md +35 -0
- package/RNImageStitcher.podspec +8 -1
- package/android/build.gradle +0 -16
- package/android/src/main/cpp/CMakeLists.txt +2 -63
- package/android/src/main/cpp/image_stitcher_jni.cpp +14 -0
- package/android/src/main/cpp/keyframe_gate_jni.cpp +13 -0
- package/android/src/main/java/io/imagestitcher/rn/BatchStitcher.kt +285 -3
- package/android/src/main/java/io/imagestitcher/rn/IncrementalStitcher.kt +180 -1162
- package/android/src/main/java/io/imagestitcher/rn/KeyframeGate.kt +29 -0
- package/android/src/main/java/io/imagestitcher/rn/RNImageStitcherPackage.kt +0 -4
- package/android/src/main/java/io/imagestitcher/rn/RNSARCameraView.kt +13 -64
- package/cpp/keyframe_gate.cpp +82 -23
- package/cpp/keyframe_gate.hpp +31 -2
- package/cpp/stitcher.cpp +208 -28
- package/cpp/tests/CMakeLists.txt +18 -12
- package/cpp/tests/keyframe_timebudget_test.cpp +65 -0
- package/cpp/tests/warp_guard_test.cpp +48 -0
- package/cpp/warp_guard.hpp +41 -0
- package/dist/ar/useARSession.d.ts +9 -0
- package/dist/ar/useARSession.js +24 -2
- package/dist/camera/Camera.d.ts +31 -16
- package/dist/camera/Camera.js +27 -4
- package/dist/camera/CaptureStitchStatsToast.d.ts +15 -2
- package/dist/camera/CaptureStitchStatsToast.js +27 -7
- package/dist/camera/PanoramaSettings.d.ts +10 -223
- package/dist/camera/PanoramaSettings.js +6 -28
- package/dist/camera/PanoramaSettingsBridge.d.ts +1 -24
- package/dist/camera/PanoramaSettingsBridge.js +3 -102
- package/dist/camera/PanoramaSettingsModal.js +7 -1
- package/dist/camera/buildPanoramaInitialSettings.d.ts +11 -0
- package/dist/camera/buildPanoramaInitialSettings.js +4 -0
- package/dist/camera/cameraErrorMessages.d.ts +32 -0
- package/dist/camera/cameraErrorMessages.js +53 -0
- package/dist/camera/selectCaptureDevice.d.ts +5 -1
- package/dist/camera/selectCaptureDevice.js +22 -2
- package/dist/camera/useCapture.js +38 -0
- package/dist/index.d.ts +5 -8
- package/dist/index.js +11 -34
- package/dist/stitching/incremental.d.ts +1 -117
- package/dist/stitching/stitchVideo.d.ts +0 -35
- package/dist/types.d.ts +0 -87
- package/ios/Sources/RNImageStitcher/IncrementalStitcher.swift +96 -674
- package/ios/Sources/RNImageStitcher/IncrementalStitcherBridge.swift +9 -12
- package/ios/Sources/RNImageStitcher/KeyframeGate.swift +14 -0
- package/ios/Sources/RNImageStitcher/KeyframeGateBridge.h +7 -0
- package/ios/Sources/RNImageStitcher/KeyframeGateBridge.mm +6 -0
- package/ios/Sources/RNImageStitcher/OpenCVKeyframeCollector.h +2 -2
- package/ios/Sources/RNImageStitcher/OpenCVKeyframeCollector.mm +3 -3
- package/ios/Sources/RNImageStitcher/OpenCVStitcher.h +28 -60
- package/ios/Sources/RNImageStitcher/OpenCVStitcher.mm +180 -921
- package/ios/Sources/RNImageStitcher/RNSARSession.swift +10 -35
- package/ios/Sources/RNImageStitcher/Stitcher.swift +84 -35
- package/ios/Sources/RNImageStitcher/StitcherBridge.m +13 -0
- package/ios/Sources/RNImageStitcher/StitcherBridge.swift +132 -5
- package/package.json +3 -2
- package/src/ar/useARSession.ts +35 -5
- package/src/camera/Camera.tsx +63 -24
- package/src/camera/CaptureStitchStatsToast.tsx +58 -14
- package/src/camera/PanoramaSettings.ts +16 -289
- package/src/camera/PanoramaSettingsBridge.ts +3 -114
- package/src/camera/PanoramaSettingsModal.tsx +14 -1
- package/src/camera/__tests__/PanoramaSettingsBridge.test.ts +3 -188
- package/src/camera/__tests__/buildPanoramaInitialSettings.test.ts +41 -0
- package/src/camera/__tests__/cameraErrorMessages.test.ts +76 -0
- package/src/camera/__tests__/selectCaptureDevice.test.ts +33 -0
- package/src/camera/buildPanoramaInitialSettings.ts +17 -0
- package/src/camera/cameraErrorMessages.ts +84 -0
- package/src/camera/selectCaptureDevice.ts +28 -3
- package/src/camera/useCapture.ts +44 -1
- package/src/index.ts +11 -40
- package/src/stitching/incremental.ts +3 -140
- package/src/stitching/stitchVideo.ts +0 -26
- package/src/types.ts +0 -95
- package/android/src/main/cpp/stitcher_jsi_install_jni.cpp +0 -227
- package/android/src/main/java/io/imagestitcher/rn/IncrementalFirstwinsEngine.kt +0 -1081
- package/android/src/main/java/io/imagestitcher/rn/StitcherJsiInstallerModule.kt +0 -103
- package/android/src/main/java/io/imagestitcher/rn/StitcherWorkletRuntime.kt +0 -256
- package/cpp/stitcher_frame_jsi.cpp +0 -214
- package/cpp/stitcher_frame_jsi.hpp +0 -108
- package/cpp/stitcher_proxy_jsi.cpp +0 -109
- package/cpp/stitcher_proxy_jsi.hpp +0 -46
- package/cpp/stitcher_worklet_dispatch.cpp +0 -103
- package/cpp/stitcher_worklet_dispatch.hpp +0 -71
- package/cpp/stitcher_worklet_registry.cpp +0 -91
- package/cpp/stitcher_worklet_registry.hpp +0 -146
- package/cpp/tests/stitcher_worklet_registry_test.cpp +0 -195
- package/dist/stitching/IncrementalStitcherView.d.ts +0 -41
- package/dist/stitching/IncrementalStitcherView.js +0 -157
- package/dist/stitching/StitcherWorkletRegistry.d.ts +0 -117
- package/dist/stitching/StitcherWorkletRegistry.js +0 -78
- package/dist/stitching/ensureStitcherProxyInstalled.d.ts +0 -8
- package/dist/stitching/ensureStitcherProxyInstalled.js +0 -81
- package/dist/stitching/useFrameProcessor.d.ts +0 -119
- package/dist/stitching/useFrameProcessor.js +0 -196
- package/dist/stitching/useFrameStream.d.ts +0 -34
- package/dist/stitching/useFrameStream.js +0 -234
- package/dist/stitching/useThrottledFrameProcessor.d.ts +0 -33
- package/dist/stitching/useThrottledFrameProcessor.js +0 -132
- package/ios/Sources/RNImageStitcher/OpenCVIncrementalStitcher.h +0 -474
- package/ios/Sources/RNImageStitcher/OpenCVIncrementalStitcher.mm +0 -1328
- package/ios/Sources/RNImageStitcher/OpenCVSlitScanStitcher.h +0 -103
- package/ios/Sources/RNImageStitcher/OpenCVSlitScanStitcher.mm +0 -3285
- package/ios/Sources/RNImageStitcher/RNSARWorkletRuntime.h +0 -128
- package/ios/Sources/RNImageStitcher/RNSARWorkletRuntime.mm +0 -313
- package/ios/Sources/RNImageStitcher/SaveFrameAsJpegPlugin.mm +0 -185
- package/ios/Sources/RNImageStitcher/StitcherFrameHostObject.h +0 -60
- package/ios/Sources/RNImageStitcher/StitcherFrameHostObject.mm +0 -214
- package/ios/Sources/RNImageStitcher/StitcherJsiInstaller.h +0 -42
- package/ios/Sources/RNImageStitcher/StitcherJsiInstaller.mm +0 -160
- package/src/stitching/IncrementalStitcherView.tsx +0 -198
- package/src/stitching/StitcherWorkletRegistry.ts +0 -156
- package/src/stitching/__tests__/StitcherWorkletRegistry.test.ts +0 -176
- package/src/stitching/__tests__/ensureStitcherProxyInstalled.test.ts +0 -94
- package/src/stitching/__tests__/useThrottledFrameProcessor.test.ts +0 -178
- package/src/stitching/ensureStitcherProxyInstalled.ts +0 -141
- package/src/stitching/useFrameProcessor.ts +0 -226
- package/src/stitching/useFrameStream.ts +0 -271
- package/src/stitching/useThrottledFrameProcessor.ts +0 -145
|
@@ -140,6 +140,23 @@ internal class KeyframeGate : AutoCloseable {
|
|
|
140
140
|
nativeSetFlowMaxTranslationM(nativeHandle, value)
|
|
141
141
|
}
|
|
142
142
|
|
|
143
|
+
/// Wall-clock keyframe-interval budget, in MILLISECONDS, between
|
|
144
|
+
/// consecutive accepted keyframes before force-acceptance. Same
|
|
145
|
+
/// knob iOS exposes via setMaxKeyframeIntervalMs (KeyframeGate.swift
|
|
146
|
+
/// `maxKeyframeIntervalMs`). Unlike flowMaxTranslationM this applies
|
|
147
|
+
/// to BOTH the Pose and Flow strategies, and is passed STRAIGHT
|
|
148
|
+
/// THROUGH (already in the unit the C++ expects — no conversion).
|
|
149
|
+
/// Default 2000 ms (matches iOS); 0 = disabled. The C++ setter
|
|
150
|
+
/// clamps to ≥ 0. NOTE: like every other facade property the
|
|
151
|
+
/// initializer below does NOT fire this setter, so the caller
|
|
152
|
+
/// (IncrementalStitcher.kt) writes it explicitly at capture start
|
|
153
|
+
/// to push the value into C++ (same contract as the iOS facade).
|
|
154
|
+
var maxKeyframeIntervalMs: Double = 2000.0
|
|
155
|
+
set(value) {
|
|
156
|
+
field = value
|
|
157
|
+
nativeSetMaxKeyframeIntervalMs(nativeHandle, value)
|
|
158
|
+
}
|
|
159
|
+
|
|
143
160
|
/// 2026-05-22 (audit F5) — Flow strategy: Shi-Tomasi max corners
|
|
144
161
|
/// to track per frame. Same knob iOS exposes via setFlowMaxCorners.
|
|
145
162
|
/// C++ clamps to ≥ 30. Higher = more sensitive to fine detail but
|
|
@@ -305,6 +322,9 @@ internal class KeyframeGate : AutoCloseable {
|
|
|
305
322
|
private external fun nativeSetDisableAngularFallback(handle: Long, disabled: Boolean)
|
|
306
323
|
private external fun nativeSetFlowNoveltyPercentile(handle: Long, percentile: Double)
|
|
307
324
|
private external fun nativeSetFlowMaxTranslationM(handle: Long, metres: Double)
|
|
325
|
+
// Wall-clock keyframe-interval budget (ms). iOS parity:
|
|
326
|
+
// KeyframeGateBridge.setMaxKeyframeIntervalMs.
|
|
327
|
+
private external fun nativeSetMaxKeyframeIntervalMs(handle: Long, ms: Double)
|
|
308
328
|
// 2026-05-22 (audit F5) — flow-strategy tunables that were
|
|
309
329
|
// previously iOS-only. Add Android JNI parity so the Settings UI
|
|
310
330
|
// sliders work on both platforms.
|
|
@@ -362,6 +382,15 @@ internal class KeyframeGate : AutoCloseable {
|
|
|
362
382
|
9 -> "max-reached"
|
|
363
383
|
10 -> "overlap-too-high"
|
|
364
384
|
11 -> "overlap-too-high (angular)"
|
|
385
|
+
// Flow-strategy reasons (v0.3.0, cpp KeyframeGateDecisionReason
|
|
386
|
+
// 12-15) — strings must match the cpp/iOS labels exactly.
|
|
387
|
+
12 -> "ok-flow"
|
|
388
|
+
13 -> "first-flow"
|
|
389
|
+
14 -> "overlap-too-high (flow)"
|
|
390
|
+
15 -> "ok-flow-translation"
|
|
391
|
+
// Wall-clock keyframe-interval force-accept (Pose + Flow);
|
|
392
|
+
// cpp KeyframeGateDecisionReason::AcceptTimeInterval = 16.
|
|
393
|
+
16 -> "ok-time-interval"
|
|
365
394
|
else -> "unknown($code)"
|
|
366
395
|
}
|
|
367
396
|
}
|
|
@@ -101,10 +101,6 @@ class RNImageStitcherPackage : ReactPackage {
|
|
|
101
101
|
RNSARSession(reactContext),
|
|
102
102
|
IncrementalStitcher(reactContext),
|
|
103
103
|
FileBridge(reactContext),
|
|
104
|
-
// v0.8.0 Phase 4b.ii — Android JSI installer for the
|
|
105
|
-
// host-worklet `__stitcherProxy` global. Mirror of
|
|
106
|
-
// iOS' `StitcherJsiInstaller`.
|
|
107
|
-
StitcherJsiInstallerModule(reactContext),
|
|
108
104
|
)
|
|
109
105
|
}
|
|
110
106
|
|
|
@@ -296,20 +296,12 @@ class RNSARCameraView @JvmOverloads constructor(
|
|
|
296
296
|
// contract was already in place for Phase 4.
|
|
297
297
|
appendPose(camera, frame.timestamp)
|
|
298
298
|
|
|
299
|
-
// Forward to the incremental stitcher
|
|
300
|
-
//
|
|
301
|
-
//
|
|
302
|
-
//
|
|
303
|
-
//
|
|
304
|
-
|
|
305
|
-
//
|
|
306
|
-
// `hasHostWorklets()` is a microsecond atomic-read on the
|
|
307
|
-
// native registry — cheap enough to hit per frame. When
|
|
308
|
-
// no host worklets are registered AND no capture is active,
|
|
309
|
-
// the entire forwardToIncremental branch (including the
|
|
310
|
-
// ~3-5ms NV21 pack) is skipped — same cost envelope as
|
|
311
|
-
// before Phase 4b.iii.
|
|
312
|
-
if (ingestActive || StitcherWorkletRuntime.hasHostWorklets()) {
|
|
299
|
+
// Forward to the incremental stitcher only when capture is
|
|
300
|
+
// engaged. (The v0.8.0 host-worklet dispatch — which also
|
|
301
|
+
// forwarded preview frames whenever host worklets were
|
|
302
|
+
// registered — was archived in the 2026-06 batch-keyframe
|
|
303
|
+
// cleanup.)
|
|
304
|
+
if (ingestActive) {
|
|
313
305
|
forwardToIncremental(frame, camera)
|
|
314
306
|
}
|
|
315
307
|
|
|
@@ -541,22 +533,14 @@ class RNSARCameraView @JvmOverloads constructor(
|
|
|
541
533
|
// written to `tmpJpegFile`, passed as `legacyJpegPath`.
|
|
542
534
|
// See the v0.3 / F8.6 entries in CHANGELOG.md.)
|
|
543
535
|
//
|
|
544
|
-
//
|
|
545
|
-
//
|
|
546
|
-
//
|
|
547
|
-
//
|
|
548
|
-
//
|
|
549
|
-
//
|
|
550
|
-
//
|
|
551
|
-
// lambda before ARCore recycles the Image.
|
|
552
|
-
StitcherWorkletRuntime.installIfNeeded()
|
|
553
|
-
// v0.8.0 Phase 4b.iii — only run first-party stitching when
|
|
554
|
-
// the host has actively engaged capture (`setIncrementalIngestionActive(true)`).
|
|
555
|
-
// The host-worklet dispatch below runs regardless, so AR-mode
|
|
556
|
-
// preview frames stream through registered host worklets even
|
|
557
|
-
// before/after capture.
|
|
536
|
+
// Synchronous engine ingest. The ARCore Image ownership
|
|
537
|
+
// contract requires the engine to consume the TransferredNV21
|
|
538
|
+
// before ARCore recycles the Image, so this runs inline. Only
|
|
539
|
+
// ingest when the host has actively engaged capture
|
|
540
|
+
// (`setIncrementalIngestionActive(true)`). (The v0.8.0 worklet-
|
|
541
|
+
// runtime `runFirstParty` indirection + host-worklet fan-out
|
|
542
|
+
// were archived in the 2026-06 batch-keyframe cleanup.)
|
|
558
543
|
if (ingestActive) {
|
|
559
|
-
StitcherWorkletRuntime.runFirstParty {
|
|
560
544
|
module.ingestFromARCameraView(
|
|
561
545
|
tx = tArr[0].toDouble(),
|
|
562
546
|
ty = tArr[1].toDouble(),
|
|
@@ -608,42 +592,7 @@ class RNSARCameraView @JvmOverloads constructor(
|
|
|
608
592
|
) != null
|
|
609
593
|
},
|
|
610
594
|
)
|
|
611
|
-
} // closes StitcherWorkletRuntime.runFirstParty { … } (v0.8.0 Phase 3c)
|
|
612
595
|
} // closes `if (ingestActive)` (v0.8.0 Phase 4b.iii)
|
|
613
|
-
|
|
614
|
-
// ── v0.8.0 Phase 4b.iii — host-worklet fan-out ─────────────
|
|
615
|
-
//
|
|
616
|
-
// Dispatch the AR frame to every host worklet registered via
|
|
617
|
-
// `globalThis.__stitcherProxy.install(workletFn)` (the
|
|
618
|
-
// `useFrameProcessor` hook's AR-mode path). The native side
|
|
619
|
-
// fast-path early-exits when the registry is empty (~ns
|
|
620
|
-
// cost), so this call is free for first-party-only deployments.
|
|
621
|
-
//
|
|
622
|
-
// Map the trackingState back to the JS-visible string set.
|
|
623
|
-
// `RNSARSession.TRACKING_*` are int codes; we re-derive the
|
|
624
|
-
// string here instead of plumbing it through. (Could be
|
|
625
|
-
// refactored into a helper if/when other call sites need
|
|
626
|
-
// it.)
|
|
627
|
-
val trackingStateStr = when (camera.trackingState) {
|
|
628
|
-
TrackingState.TRACKING -> "normal"
|
|
629
|
-
TrackingState.PAUSED -> "limited"
|
|
630
|
-
TrackingState.STOPPED -> "notAvailable"
|
|
631
|
-
else -> ""
|
|
632
|
-
}
|
|
633
|
-
StitcherWorkletRuntime.dispatchToHostWorklets(
|
|
634
|
-
nv21Bytes = packed.nv21,
|
|
635
|
-
width = packed.width,
|
|
636
|
-
height = packed.height,
|
|
637
|
-
qx = qarr[0].toDouble(),
|
|
638
|
-
qy = qarr[1].toDouble(),
|
|
639
|
-
qz = qarr[2].toDouble(),
|
|
640
|
-
qw = qarr[3].toDouble(),
|
|
641
|
-
tx = tArr[0].toDouble(),
|
|
642
|
-
ty = tArr[1].toDouble(),
|
|
643
|
-
tz = tArr[2].toDouble(),
|
|
644
|
-
timestampNs = frame.timestamp.toDouble(),
|
|
645
|
-
trackingState = trackingStateStr,
|
|
646
|
-
)
|
|
647
596
|
}
|
|
648
597
|
|
|
649
598
|
/// v0.13.2 — map the JS physical device orientation to the
|
package/cpp/keyframe_gate.cpp
CHANGED
|
@@ -28,6 +28,7 @@
|
|
|
28
28
|
#include "keyframe_gate.hpp"
|
|
29
29
|
|
|
30
30
|
#include <algorithm>
|
|
31
|
+
#include <chrono>
|
|
31
32
|
#include <cmath>
|
|
32
33
|
#include <cstring>
|
|
33
34
|
#include <cstdint>
|
|
@@ -285,6 +286,10 @@ struct KeyframeGate::Impl {
|
|
|
285
286
|
// only matters when the gate is used WITHOUT the JS bridge.
|
|
286
287
|
double overlapThreshold = 0.2;
|
|
287
288
|
int32_t maxCount = 6;
|
|
289
|
+
/// Time-budget force-accept (both strategies). > 0 ms = force a
|
|
290
|
+
/// keyframe when this much wall-clock time has elapsed since the last
|
|
291
|
+
/// accept, even if novelty < threshold. 0.0 = disabled. See hpp.
|
|
292
|
+
double maxKeyframeIntervalMs = 0.0;
|
|
288
293
|
|
|
289
294
|
// V16 A2 — strategy + flow tunables. Default is Pose to keep
|
|
290
295
|
// pre-A2 behaviour for any caller that hasn't switched. The
|
|
@@ -327,6 +332,13 @@ struct KeyframeGate::Impl {
|
|
|
327
332
|
std::optional<PlaneBasis> planeForCapture;
|
|
328
333
|
bool forceAcceptNext = false;
|
|
329
334
|
std::optional<Pose> lastAcceptedPose;
|
|
335
|
+
/// Time-budget state. `lastAcceptSteadyMs` = monotonic ms stamp of
|
|
336
|
+
/// the last accepted keyframe (-1 until the first accept).
|
|
337
|
+
/// `currentNowMs` = stamp for the in-flight evaluate() call, stashed
|
|
338
|
+
/// at entry so the static angular-fallback + the strategy paths can
|
|
339
|
+
/// read it without threading it through every signature.
|
|
340
|
+
int64_t lastAcceptSteadyMs = -1;
|
|
341
|
+
int64_t currentNowMs = -1;
|
|
330
342
|
|
|
331
343
|
// ── Flow-path state (V16 A2) ──────────────────────────────────
|
|
332
344
|
// `prevFrameGray` is the WORKING-RESOLUTION grayscale image of the
|
|
@@ -380,6 +392,9 @@ void KeyframeGate::setFlowMinDistance(double d) { pImpl_->flowMinDistanc
|
|
|
380
392
|
// V16 — translation budget. Clamp to non-negative; 0.0 disables the
|
|
381
393
|
// force-accept entirely (callers can opt-out by passing 0).
|
|
382
394
|
void KeyframeGate::setFlowMaxTranslationM(double m) { pImpl_->flowMaxTranslationM = (m < 0.0 ? 0.0 : m); }
|
|
395
|
+
// Time-budget force-accept (both strategies). Clamp to non-negative;
|
|
396
|
+
// 0.0 disables it (callers opt out by passing 0).
|
|
397
|
+
void KeyframeGate::setMaxKeyframeIntervalMs(double ms) { pImpl_->maxKeyframeIntervalMs = (ms < 0.0 ? 0.0 : ms); }
|
|
383
398
|
// V16 — novelty percentile. Clamp to [0.5, 0.99]. Below 0.5 the
|
|
384
399
|
// estimate becomes too sensitive to the BEST-tracked-features (under-
|
|
385
400
|
// reports user-perceived novelty); above 0.99 it's effectively max-
|
|
@@ -395,6 +410,8 @@ void KeyframeGate::reset() {
|
|
|
395
410
|
pImpl_->planeForCapture.reset();
|
|
396
411
|
pImpl_->forceAcceptNext = false;
|
|
397
412
|
pImpl_->lastAcceptedPose.reset();
|
|
413
|
+
pImpl_->lastAcceptSteadyMs = -1;
|
|
414
|
+
pImpl_->currentNowMs = -1;
|
|
398
415
|
// V16 A2 — drop flow state. release() returns the cv::Mat to
|
|
399
416
|
// empty (refcount-managed); std::vector::clear() is the
|
|
400
417
|
// canonical empty. Mandatory: leftover state from a prior
|
|
@@ -428,6 +445,14 @@ bool KeyframeGate::isEnabled() const { return pImpl_->enabled; }
|
|
|
428
445
|
// remain in the enum for back-compat but are NO LONGER EMITTED.
|
|
429
446
|
// Diagnostic logging at the call sites tells us if a degenerate
|
|
430
447
|
// projection triggered the fallback.
|
|
448
|
+
// Monotonic wall-clock in milliseconds for the time-budget force-accept.
|
|
449
|
+
// Used when the caller passes monotonicNowMs < 0 (production); tests pass
|
|
450
|
+
// an explicit stamp to drive elapsed time deterministically.
|
|
451
|
+
static int64_t steadyNowMs() {
|
|
452
|
+
return std::chrono::duration_cast<std::chrono::milliseconds>(
|
|
453
|
+
std::chrono::steady_clock::now().time_since_epoch()).count();
|
|
454
|
+
}
|
|
455
|
+
|
|
431
456
|
KeyframeGateDecision KeyframeGate::evaluateAngularFallback(
|
|
432
457
|
Impl& s,
|
|
433
458
|
const Pose& pose)
|
|
@@ -441,6 +466,16 @@ KeyframeGateDecision KeyframeGate::evaluateAngularFallback(
|
|
|
441
466
|
return { false, KeyframeGateDecisionReason::RejectMaxReached,
|
|
442
467
|
-1.0, s.acceptedCount, s.maxCount };
|
|
443
468
|
}
|
|
469
|
+
// Time-budget force-accept — overrides BOTH the disable-angular hard
|
|
470
|
+
// reject and the novelty reject below. The angular path keeps no image
|
|
471
|
+
// baseline (only lastAcceptedPose), so this is a complete accept.
|
|
472
|
+
if (timeBudgetCrossed(s.maxKeyframeIntervalMs, s.lastAcceptSteadyMs, s.currentNowMs)) {
|
|
473
|
+
s.lastAcceptedPose = pose;
|
|
474
|
+
s.lastAcceptSteadyMs = s.currentNowMs;
|
|
475
|
+
s.acceptedCount += 1;
|
|
476
|
+
return { true, KeyframeGateDecisionReason::AcceptTimeInterval,
|
|
477
|
+
-1.0, s.acceptedCount, s.maxCount };
|
|
478
|
+
}
|
|
444
479
|
// 2026-05-14 — non-AR-mode opt-out. When `disableAngularFallback`
|
|
445
480
|
// is set, treat every angular-fallback call as a hard reject.
|
|
446
481
|
// The caller's flow strategy is then the only path that can
|
|
@@ -469,6 +504,7 @@ KeyframeGateDecision KeyframeGate::evaluateAngularFallback(
|
|
|
469
504
|
newContent, s.acceptedCount, s.maxCount };
|
|
470
505
|
}
|
|
471
506
|
s.lastAcceptedPose = pose;
|
|
507
|
+
s.lastAcceptSteadyMs = s.currentNowMs;
|
|
472
508
|
s.acceptedCount += 1;
|
|
473
509
|
return { true, KeyframeGateDecisionReason::AcceptOkAngular,
|
|
474
510
|
newContent, s.acceptedCount, s.maxCount };
|
|
@@ -476,9 +512,14 @@ KeyframeGateDecision KeyframeGate::evaluateAngularFallback(
|
|
|
476
512
|
|
|
477
513
|
|
|
478
514
|
KeyframeGateDecision KeyframeGate::evaluate(const Pose& pose,
|
|
479
|
-
const PlaneTransform* latchedPlane
|
|
515
|
+
const PlaneTransform* latchedPlane,
|
|
516
|
+
int64_t monotonicNowMs)
|
|
480
517
|
{
|
|
481
518
|
Impl& s = *pImpl_;
|
|
519
|
+
// Stash the monotonic stamp for this call so the time-budget checks
|
|
520
|
+
// (here, the angular fallback, and the strategy paths) all read one
|
|
521
|
+
// consistent "now". Caller may inject it (tests); else steady_clock.
|
|
522
|
+
s.currentNowMs = (monotonicNowMs >= 0) ? monotonicNowMs : steadyNowMs();
|
|
482
523
|
|
|
483
524
|
// 1) Mode disabled → pass-through.
|
|
484
525
|
if (!s.enabled) {
|
|
@@ -504,6 +545,7 @@ KeyframeGateDecision KeyframeGate::evaluate(const Pose& pose,
|
|
|
504
545
|
}
|
|
505
546
|
}
|
|
506
547
|
s.lastAcceptedPose = pose;
|
|
548
|
+
s.lastAcceptSteadyMs = s.currentNowMs;
|
|
507
549
|
s.acceptedCount += 1;
|
|
508
550
|
return { true, KeyframeGateDecisionReason::AcceptForceLast,
|
|
509
551
|
-1.0, s.acceptedCount, s.maxCount };
|
|
@@ -512,6 +554,7 @@ KeyframeGateDecision KeyframeGate::evaluate(const Pose& pose,
|
|
|
512
554
|
// 3) First-frame anchor — always accepted.
|
|
513
555
|
if (s.acceptedCount == 0) {
|
|
514
556
|
s.lastAcceptedPose = pose;
|
|
557
|
+
s.lastAcceptSteadyMs = s.currentNowMs;
|
|
515
558
|
if (latchedPlane) {
|
|
516
559
|
auto basis = planeBasisFromMatrix(latchedPlane->m);
|
|
517
560
|
if (basis) {
|
|
@@ -587,16 +630,22 @@ KeyframeGateDecision KeyframeGate::evaluate(const Pose& pose,
|
|
|
587
630
|
if (overlapRatio > 1.0f) overlapRatio = 1.0f;
|
|
588
631
|
double newContentFraction = 1.0 - static_cast<double>(overlapRatio);
|
|
589
632
|
|
|
590
|
-
|
|
633
|
+
const bool poseTimeCrossed = timeBudgetCrossed(
|
|
634
|
+
s.maxKeyframeIntervalMs, s.lastAcceptSteadyMs, s.currentNowMs);
|
|
635
|
+
if (newContentFraction < s.overlapThreshold && !poseTimeCrossed) {
|
|
591
636
|
return { false, KeyframeGateDecisionReason::RejectOverlapTooHigh,
|
|
592
637
|
newContentFraction, s.acceptedCount, s.maxCount };
|
|
593
638
|
}
|
|
594
639
|
|
|
595
|
-
// Accept.
|
|
640
|
+
// Accept (novelty crossed, or the time-budget forced it).
|
|
596
641
|
s.lastCornersOnPlane = currentCorners;
|
|
597
642
|
s.lastAcceptedPose = pose;
|
|
643
|
+
s.lastAcceptSteadyMs = s.currentNowMs;
|
|
598
644
|
s.acceptedCount += 1;
|
|
599
|
-
return { true,
|
|
645
|
+
return { true,
|
|
646
|
+
(newContentFraction >= s.overlapThreshold)
|
|
647
|
+
? KeyframeGateDecisionReason::AcceptOk
|
|
648
|
+
: KeyframeGateDecisionReason::AcceptTimeInterval,
|
|
600
649
|
newContentFraction, s.acceptedCount, s.maxCount };
|
|
601
650
|
}
|
|
602
651
|
|
|
@@ -699,9 +748,13 @@ KeyframeGateDecision KeyframeGate::evaluateWithFrame(
|
|
|
699
748
|
const uint8_t* grayData,
|
|
700
749
|
int32_t width,
|
|
701
750
|
int32_t height,
|
|
702
|
-
int32_t stride
|
|
751
|
+
int32_t stride,
|
|
752
|
+
int64_t monotonicNowMs)
|
|
703
753
|
{
|
|
704
754
|
Impl& s = *pImpl_;
|
|
755
|
+
// Stash the monotonic stamp (see evaluate()). The Pose-strategy
|
|
756
|
+
// dispatch below forwards it so both entry points agree on "now".
|
|
757
|
+
s.currentNowMs = (monotonicNowMs >= 0) ? monotonicNowMs : steadyNowMs();
|
|
705
758
|
|
|
706
759
|
// §1 — disabled passes through unchanged for either strategy.
|
|
707
760
|
if (!s.enabled) {
|
|
@@ -717,6 +770,7 @@ KeyframeGateDecision KeyframeGate::evaluateWithFrame(
|
|
|
717
770
|
// mostly defensive.
|
|
718
771
|
if (s.forceAcceptNext) {
|
|
719
772
|
s.forceAcceptNext = false;
|
|
773
|
+
s.lastAcceptSteadyMs = s.currentNowMs;
|
|
720
774
|
s.acceptedCount += 1;
|
|
721
775
|
// No newContent fraction — we accepted unconditionally.
|
|
722
776
|
return { true, KeyframeGateDecisionReason::AcceptForceLast,
|
|
@@ -728,7 +782,7 @@ KeyframeGateDecision KeyframeGate::evaluateWithFrame(
|
|
|
728
782
|
// Pose path is OpenCV-free and identical to the
|
|
729
783
|
// backward-compat `evaluate()` entry point. Skip the
|
|
730
784
|
// grayscale wrap entirely — `grayData` is ignored.
|
|
731
|
-
return evaluate(pose, latchedPlane);
|
|
785
|
+
return evaluate(pose, latchedPlane, s.currentNowMs);
|
|
732
786
|
}
|
|
733
787
|
|
|
734
788
|
// Flow path — wrap incoming pixel data as a non-owning cv::Mat
|
|
@@ -738,7 +792,7 @@ KeyframeGateDecision KeyframeGate::evaluateWithFrame(
|
|
|
738
792
|
// Defensive: caller forgot to supply image data despite
|
|
739
793
|
// strategy=Flow. Fall back to pose path so we degrade
|
|
740
794
|
// gracefully rather than crashing on a null deref.
|
|
741
|
-
return evaluate(pose, latchedPlane);
|
|
795
|
+
return evaluate(pose, latchedPlane, s.currentNowMs);
|
|
742
796
|
}
|
|
743
797
|
cv::Mat currGrayFull(height, width, CV_8UC1,
|
|
744
798
|
const_cast<uint8_t*>(grayData),
|
|
@@ -760,6 +814,7 @@ KeyframeGateDecision KeyframeGate::evaluateWithFrame(
|
|
|
760
814
|
s.prevFrameOrigWidth = width;
|
|
761
815
|
s.prevFrameOrigHeight = height;
|
|
762
816
|
s.lastAcceptedPose = pose;
|
|
817
|
+
s.lastAcceptSteadyMs = s.currentNowMs;
|
|
763
818
|
s.acceptedCount = 1;
|
|
764
819
|
return { true, KeyframeGateDecisionReason::AcceptFirstFlow,
|
|
765
820
|
-1.0, s.acceptedCount, s.maxCount };
|
|
@@ -878,24 +933,27 @@ KeyframeGateDecision KeyframeGate::evaluateWithFrame(
|
|
|
878
933
|
const bool translationBudgetCrossed =
|
|
879
934
|
(s.flowMaxTranslationM > 0.0) &&
|
|
880
935
|
(translationSinceLastAccept >= s.flowMaxTranslationM);
|
|
881
|
-
|
|
882
|
-
//
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
//
|
|
887
|
-
//
|
|
888
|
-
|
|
936
|
+
// Time-budget force-accept (both strategies; see hpp). Same shape as
|
|
937
|
+
// the translation budget, but measured on elapsed wall-clock time.
|
|
938
|
+
const bool flowTimeCrossed = timeBudgetCrossed(
|
|
939
|
+
s.maxKeyframeIntervalMs, s.lastAcceptSteadyMs, s.currentNowMs);
|
|
940
|
+
|
|
941
|
+
// §7 — accept-or-reject combined check. Accept if the novelty crossed
|
|
942
|
+
// `overlapThreshold` (the original rule) OR the translation budget OR
|
|
943
|
+
// the time budget was exceeded (the force-accepts). The decision
|
|
944
|
+
// reason distinguishes them so telemetry can identify what drove each.
|
|
945
|
+
if (novelty < s.overlapThreshold && !translationBudgetCrossed && !flowTimeCrossed) {
|
|
889
946
|
return { false, KeyframeGateDecisionReason::RejectOverlapTooHighFlow,
|
|
890
947
|
novelty, s.acceptedCount, s.maxCount };
|
|
891
948
|
}
|
|
892
|
-
// Pick the reason —
|
|
893
|
-
//
|
|
894
|
-
// novelty path is the "natural" reason).
|
|
949
|
+
// Pick the reason — precedence: natural novelty, then translation, then
|
|
950
|
+
// time (if several crossed, report the most "natural" cause).
|
|
895
951
|
const KeyframeGateDecisionReason acceptReason =
|
|
896
952
|
(novelty >= s.overlapThreshold)
|
|
897
953
|
? KeyframeGateDecisionReason::AcceptOkFlow
|
|
898
|
-
:
|
|
954
|
+
: translationBudgetCrossed
|
|
955
|
+
? KeyframeGateDecisionReason::AcceptFlowTranslation
|
|
956
|
+
: KeyframeGateDecisionReason::AcceptTimeInterval;
|
|
899
957
|
|
|
900
958
|
// §8 — accept. Re-detect features in the newly-accepted frame
|
|
901
959
|
// (the previous set is now stale; many of them have moved out
|
|
@@ -915,11 +973,12 @@ KeyframeGateDecision KeyframeGate::evaluateWithFrame(
|
|
|
915
973
|
s.prevFrameOrigWidth = width;
|
|
916
974
|
s.prevFrameOrigHeight = height;
|
|
917
975
|
s.lastAcceptedPose = pose;
|
|
976
|
+
s.lastAcceptSteadyMs = s.currentNowMs;
|
|
918
977
|
s.acceptedCount += 1;
|
|
919
|
-
// `acceptReason` was decided in §7 —
|
|
920
|
-
//
|
|
921
|
-
//
|
|
922
|
-
// distinguish.
|
|
978
|
+
// `acceptReason` was decided in §7 — AcceptOkFlow (novelty crossed),
|
|
979
|
+
// AcceptFlowTranslation (translation budget forced it), or
|
|
980
|
+
// AcceptTimeInterval (time budget forced it). Reported back here so
|
|
981
|
+
// the host's telemetry can distinguish.
|
|
923
982
|
return { true, acceptReason,
|
|
924
983
|
novelty, s.acceptedCount, s.maxCount };
|
|
925
984
|
}
|
package/cpp/keyframe_gate.hpp
CHANGED
|
@@ -99,8 +99,23 @@ enum class KeyframeGateDecisionReason : int32_t {
|
|
|
99
99
|
AcceptFirstFlow = 13, // "first-flow" — first frame under flow strategy
|
|
100
100
|
RejectOverlapTooHighFlow = 14, // "overlap-too-high (flow)"
|
|
101
101
|
AcceptFlowTranslation = 15, // "ok-flow-translation" — translation since last accept exceeded flowMaxTranslationM (force-accept even when novelty < threshold)
|
|
102
|
+
AcceptTimeInterval = 16, // "ok-time-interval" — wall-clock interval since last accept exceeded maxKeyframeIntervalMs (force-accept even when novelty < threshold; applies to BOTH Pose and Flow strategies)
|
|
102
103
|
};
|
|
103
104
|
|
|
105
|
+
/// Pure, OpenCV-free predicate for the time-budget force-accept — split
|
|
106
|
+
/// out so it can be unit-tested on the host WITHOUT linking the gate's
|
|
107
|
+
/// OpenCV-dependent .cpp. True iff a positive budget is set, a prior
|
|
108
|
+
/// accept stamp exists (lastAcceptMs >= 0), and at least that many
|
|
109
|
+
/// milliseconds have elapsed (nowMs - lastAcceptMs >= intervalMs).
|
|
110
|
+
inline bool timeBudgetCrossed(double intervalMs, int64_t lastAcceptMs, int64_t nowMs) {
|
|
111
|
+
// Compare elapsed-ms in `double` (not a truncating int64 cast of
|
|
112
|
+
// intervalMs) so a sub-millisecond budget doesn't collapse to
|
|
113
|
+
// "accept every frame".
|
|
114
|
+
return intervalMs > 0.0
|
|
115
|
+
&& lastAcceptMs >= 0
|
|
116
|
+
&& static_cast<double>(nowMs - lastAcceptMs) >= intervalMs;
|
|
117
|
+
}
|
|
118
|
+
|
|
104
119
|
struct KeyframeGateDecision {
|
|
105
120
|
bool accept;
|
|
106
121
|
KeyframeGateDecisionReason reason;
|
|
@@ -128,6 +143,13 @@ public:
|
|
|
128
143
|
void setEnabled(bool enabled);
|
|
129
144
|
void setOverlapThreshold(double threshold); // [0, 1]; default 0.4
|
|
130
145
|
void setMaxCount(int32_t maxCount); // ≥ 1; default 6
|
|
146
|
+
/// Time-budget force-accept (applies to BOTH strategies, Pose + Flow).
|
|
147
|
+
/// When > 0 and that many milliseconds of wall-clock time have elapsed
|
|
148
|
+
/// since the last accepted keyframe, the gate force-accepts the current
|
|
149
|
+
/// frame even if novelty < threshold — a "don't go longer than N ms
|
|
150
|
+
/// without a keyframe" guarantee for slow / static pans. Counts toward
|
|
151
|
+
/// maxCount (respects the cap). Default 0.0 = disabled; clamped to ≥ 0.
|
|
152
|
+
void setMaxKeyframeIntervalMs(double ms);
|
|
131
153
|
void markNextFrameAsLast(); // one-shot, consumed by next evaluate()
|
|
132
154
|
void reset(); // clears acceptedCount, lastCorners, planeCached AND flow state
|
|
133
155
|
|
|
@@ -210,14 +232,21 @@ public:
|
|
|
210
232
|
// @param stride bytes per row (usually equal to width;
|
|
211
233
|
// larger when the underlying buffer is
|
|
212
234
|
// padded).
|
|
235
|
+
// @param monotonicNowMs optional monotonic timestamp (milliseconds)
|
|
236
|
+
// for the time-budget force-accept. Pass -1
|
|
237
|
+
// (default) to have the gate read its own
|
|
238
|
+
// steady_clock; tests pass an explicit value to
|
|
239
|
+
// drive elapsed time deterministically.
|
|
213
240
|
KeyframeGateDecision evaluate(const Pose& pose,
|
|
214
|
-
const PlaneTransform* latchedPlane
|
|
241
|
+
const PlaneTransform* latchedPlane,
|
|
242
|
+
int64_t monotonicNowMs = -1);
|
|
215
243
|
KeyframeGateDecision evaluateWithFrame(const Pose& pose,
|
|
216
244
|
const PlaneTransform* latchedPlane,
|
|
217
245
|
const uint8_t* grayData,
|
|
218
246
|
int32_t width,
|
|
219
247
|
int32_t height,
|
|
220
|
-
int32_t stride
|
|
248
|
+
int32_t stride,
|
|
249
|
+
int64_t monotonicNowMs = -1);
|
|
221
250
|
|
|
222
251
|
// ── State accessors (read-only, post-evaluate) ────────────────
|
|
223
252
|
int32_t getAcceptedCount() const;
|