react-native-image-stitcher 0.8.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +269 -0
- package/android/build.gradle +10 -0
- package/android/src/main/java/io/imagestitcher/rn/IncrementalStitcher.kt +115 -10
- package/android/src/main/java/io/imagestitcher/rn/RNImageStitcherPackage.kt +17 -3
- package/android/src/main/java/io/imagestitcher/rn/SaveFrameAsJpegPlugin.kt +162 -0
- package/cpp/stitcher_worklet_registry.cpp +10 -0
- package/cpp/stitcher_worklet_registry.hpp +10 -0
- package/cpp/tests/CMakeLists.txt +98 -0
- package/cpp/tests/README.md +86 -0
- package/cpp/tests/pose_test.cpp +74 -0
- package/cpp/tests/stitcher_frame_data_test.cpp +132 -0
- package/cpp/tests/stitcher_worklet_registry_test.cpp +195 -0
- package/cpp/tests/stubs/jsi/jsi.h +33 -0
- package/cpp/tests/stubs/react-native-worklets-core/WKTJsiWorklet.h +34 -0
- package/dist/camera/useCapture.d.ts +1 -1
- package/dist/camera/useCapture.js +1 -1
- package/dist/index.d.ts +4 -0
- package/dist/index.js +20 -1
- package/dist/stitching/incremental.d.ts +41 -0
- package/dist/stitching/useFrameStream.d.ts +34 -0
- package/dist/stitching/useFrameStream.js +234 -0
- package/dist/stitching/useThrottledFrameProcessor.d.ts +33 -0
- package/dist/stitching/useThrottledFrameProcessor.js +132 -0
- package/dist/types.d.ts +87 -0
- package/ios/Sources/RNImageStitcher/IncrementalStitcher.swift +138 -9
- package/ios/Sources/RNImageStitcher/IncrementalStitcherBridge.swift +50 -14
- package/ios/Sources/RNImageStitcher/SaveFrameAsJpegPlugin.mm +185 -0
- package/package.json +1 -1
- package/src/camera/useCapture.ts +1 -1
- package/src/index.ts +19 -0
- package/src/stitching/__tests__/subscribeIncrementalState.refine.test.ts +276 -0
- package/src/stitching/__tests__/useThrottledFrameProcessor.test.ts +178 -0
- package/src/stitching/incremental.ts +42 -0
- package/src/stitching/useFrameStream.ts +271 -0
- package/src/stitching/useThrottledFrameProcessor.ts +145 -0
- package/src/types.ts +95 -0
package/CHANGELOG.md
CHANGED
|
@@ -16,6 +16,275 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
16
16
|
|
|
17
17
|
## [Unreleased]
|
|
18
18
|
|
|
19
|
+
## [0.10.0] — 2026-05-28
|
|
20
|
+
|
|
21
|
+
### Added — v0.10.0 PR A: host-side test infrastructure (`#9A` + `#11A`)
|
|
22
|
+
|
|
23
|
+
Two parallel test harnesses landed so future tech-debt PRs in the
|
|
24
|
+
v0.10.0 sweep (and beyond) can pin invariants without standing up a
|
|
25
|
+
device build per change.
|
|
26
|
+
|
|
27
|
+
#### Shared C++ Google Test runner (`#9A`)
|
|
28
|
+
|
|
29
|
+
- `cpp/tests/CMakeLists.txt` — standalone CMake project that fetches
|
|
30
|
+
Google Test `v1.14.0` via `FetchContent` and compiles a single
|
|
31
|
+
`stitcher_cpp_tests` executable. No system-wide gtest install
|
|
32
|
+
required.
|
|
33
|
+
- `scripts/run-cpp-tests.sh` — one-shot configure / build / `ctest`
|
|
34
|
+
driver. Output lands under gitignored `build/cpp-tests/`.
|
|
35
|
+
- Initial suite (17 cases):
|
|
36
|
+
- `Pose` / `PlaneTransform` POD layout, size, field-offset
|
|
37
|
+
invariants (pinned to the cross-platform marshalling contract
|
|
38
|
+
in `cpp/ar_frame_pose.h`).
|
|
39
|
+
- `StitcherFrameData` default-construction invariants the JSI
|
|
40
|
+
host-object `get()` dispatch depends on (e.g. `qw=1.0`,
|
|
41
|
+
`hasTranslation=false`).
|
|
42
|
+
- `PixelBufferReader` `copyTo` clipping contract — validated via
|
|
43
|
+
a `FakePixelBufferReader` test helper.
|
|
44
|
+
- `StitcherWorkletRegistry` lifecycle: shared-instance, install
|
|
45
|
+
/uninstall/count/snapshot, snapshot independence from later
|
|
46
|
+
mutations, concurrent installs (16 threads × 32 each) yield
|
|
47
|
+
unique IDs without lock contention bugs.
|
|
48
|
+
- New test-only registry seam `_installEntryForTests(invoker)` (in
|
|
49
|
+
`cpp/stitcher_worklet_registry.{hpp,cpp}`) — mirrors the existing
|
|
50
|
+
`_resetForTests` pattern. Bypasses the JSI runtime path so tests
|
|
51
|
+
don't need Hermes + worklets-core; `nullptr` invokers are safe
|
|
52
|
+
because the registry never dereferences them.
|
|
53
|
+
- JSI / worklets-core stubs under `cpp/tests/stubs/` let
|
|
54
|
+
`stitcher_worklet_registry.cpp` compile in the host-side test
|
|
55
|
+
target without pulling in React Native's JSI headers or the
|
|
56
|
+
worklets-core library. Stubs are scoped exclusively to the test
|
|
57
|
+
include path; production builds never see them.
|
|
58
|
+
- See `cpp/tests/README.md` for the strategy + a list of what's
|
|
59
|
+
deferred to v0.11.0+ (KeyframeGate / OpenCV-dependent code; JSI
|
|
60
|
+
host-object dispatch).
|
|
61
|
+
|
|
62
|
+
#### Android JUnit scaffold (`#11A`)
|
|
63
|
+
|
|
64
|
+
- `android/build.gradle` — adds `testImplementation
|
|
65
|
+
"junit:junit:4.13.2"`. Minimal — only JUnit 4 (matches AGP's
|
|
66
|
+
default test runner).
|
|
67
|
+
- `android/src/test/java/io/imagestitcher/rn/TransferredNV21Test.kt`
|
|
68
|
+
— 6 tests covering the v0.10.0 `TransferredNV21` single-use
|
|
69
|
+
ownership wrapper: constructor empty/non-empty, takeOnce returns
|
|
70
|
+
the original reference, takeOnce throws on second call, thread-
|
|
71
|
+
safe single-winner under 16-thread contention, distinct wrappers
|
|
72
|
+
are independent.
|
|
73
|
+
- Run via `./gradlew :react-native-image-stitcher:testDebugUnitTest`.
|
|
74
|
+
|
|
75
|
+
Neither suite changes runtime behaviour — both are additive test
|
|
76
|
+
infrastructure.
|
|
77
|
+
|
|
78
|
+
### Added — v0.10.0 PR B: `refinePanorama` progress events + cleanup audit (`#15A` + `#16C`)
|
|
79
|
+
|
|
80
|
+
#### `#15A` — phase-milestone progress emit from `refinePanorama`
|
|
81
|
+
|
|
82
|
+
`refinePanorama` (both the explicit JS `module.refinePanorama(...)` API
|
|
83
|
+
and the hybrid-engine auto-refine path that calls it internally) now
|
|
84
|
+
emits coarse phase events on the existing `IncrementalStateUpdate`
|
|
85
|
+
device-event channel. Five stages cover one refine lifetime:
|
|
86
|
+
|
|
87
|
+
| Stage | `refineProgress` | When |
|
|
88
|
+
| ------------- | ---------------- | ------------------------------------ |
|
|
89
|
+
| `validating` | 0.05 | start of method, before any I/O |
|
|
90
|
+
| `stitching` | 0.10 | OpenCV stitch in flight |
|
|
91
|
+
| `writing` | 0.90 | stitch returned, JPEG written |
|
|
92
|
+
| `done` | 1.00 | success — promise about to resolve |
|
|
93
|
+
| `error` | 1.00 | failure — `refineError` is set |
|
|
94
|
+
|
|
95
|
+
`refineStage` carries the stage string; `refineProgress` carries the
|
|
96
|
+
fraction; `refineFrames` reports the input keyframe count; `refineError`
|
|
97
|
+
is populated on the failure path so the host can render a one-line
|
|
98
|
+
failure pill.
|
|
99
|
+
|
|
100
|
+
Coarse on purpose: OpenCV's `Stitcher` doesn't expose mid-pipeline
|
|
101
|
+
progress, so the `0.10 → 0.90` jump is one opaque step. JS uses
|
|
102
|
+
`refineStage` for the UI label and `refineProgress` purely for the
|
|
103
|
+
spinner.
|
|
104
|
+
|
|
105
|
+
Reuses the existing channel (no second listener wiring required).
|
|
106
|
+
Existing JS consumers that don't read the new fields are unaffected.
|
|
107
|
+
|
|
108
|
+
#### `#16C` — moderate cleanup audit sweep
|
|
109
|
+
|
|
110
|
+
- `src/camera/useCapture.ts` — removed a stale "`useVideoCapture` (TODO)"
|
|
111
|
+
reference; the hook has existed since v0.4.
|
|
112
|
+
- `ios/Sources/RNImageStitcher/IncrementalStitcherBridge.swift` — removed
|
|
113
|
+
a self-flagged "remove this comment after" reference left over from a
|
|
114
|
+
past PiP investigation.
|
|
115
|
+
- `console.*` audit: every call in `src/` was reviewed; all 13 are
|
|
116
|
+
legitimate (warn/error for surfaceable failures; `console.info`
|
|
117
|
+
one-shots that document known tradeoffs). No removals needed.
|
|
118
|
+
- TODO/FIXME triage: 4 remaining own-code TODOs all reference tracked
|
|
119
|
+
future work (lens-probe follow-up, shared-stitcher-port-part-2,
|
|
120
|
+
EXIF writer). Left in place.
|
|
121
|
+
- `ts-prune`: 3 surface-level orphans (`PanoramaConfirmModal`,
|
|
122
|
+
`IncrementalStitcherView`, `stitchFrames`) are intentional public
|
|
123
|
+
deep-import API; not re-exported from `src/index.ts` but
|
|
124
|
+
documented and consumed by hosts. Left in place.
|
|
125
|
+
|
|
126
|
+
No production behaviour changed — these are docstring + dead-comment
|
|
127
|
+
removals only.
|
|
128
|
+
|
|
129
|
+
### Fixed — v0.10.0 PR B (iOS): refine state events not reaching JS under RN bridgeless interop
|
|
130
|
+
|
|
131
|
+
Switched `IncrementalStitcherBridge` state-event delivery from
|
|
132
|
+
`RCTEventEmitter.sendEvent` to `bridge.enqueueJSCall("RCTDeviceEventEmitter", "emit", ...)`.
|
|
133
|
+
Root cause: under RN bridgeless interop (RN 0.84), `sendEvent`
|
|
134
|
+
silently no-ops for some event-body shapes even when the bridge is
|
|
135
|
+
non-nil and the listener count is > 0 — refine events with the
|
|
136
|
+
`refineStage` / `refineProgress` / `refineFrames` keys were not
|
|
137
|
+
reaching any JS subscriber while live state events with a smaller
|
|
138
|
+
body shape on the same channel were. Also defensively
|
|
139
|
+
`removeObserver` before `addObserver` in `init()` so the
|
|
140
|
+
NotificationCenter registration is idempotent if RN re-invokes
|
|
141
|
+
`init()` on the same instance (also observed on bridgeless interop).
|
|
142
|
+
Android is unaffected — Android's bridge already emits via
|
|
143
|
+
`DeviceEventManagerModule.RCTDeviceEventEmitter.emit(...)` directly.
|
|
144
|
+
|
|
145
|
+
## [0.9.0] — 2026-05-27
|
|
146
|
+
|
|
147
|
+
### Added — layered frame-access helpers
|
|
148
|
+
|
|
149
|
+
Three new primitives completing the Tier 2 surface in the
|
|
150
|
+
three-tier extensibility pattern. See `docs/frame-access-tiers.md`
|
|
151
|
+
for the full decision flow + use-case mapping.
|
|
152
|
+
|
|
153
|
+
#### Layer 1 — `save_frame_as_jpeg` vc Frame Processor plugin (native)
|
|
154
|
+
|
|
155
|
+
Worklet-callable JPEG encoder. Registers on both platforms:
|
|
156
|
+
|
|
157
|
+
- **iOS** — `SaveFrameAsJpegPlugin.mm` (CIImage → CGImage → UIImage
|
|
158
|
+
→ UIImageJPEGRepresentation → atomic NSData write). Registered
|
|
159
|
+
via `+ (void)load` hook into `FrameProcessorPluginRegistry`.
|
|
160
|
+
- **Android** — `SaveFrameAsJpegPlugin.kt` wrapping the lib's
|
|
161
|
+
existing `YuvImageConverter.encodeJpegFromNV21` encoder (the
|
|
162
|
+
same one used by `RNSARCameraView`'s keyframe-accept callback).
|
|
163
|
+
Registered alongside `cv_flow_gate_process_frame` in
|
|
164
|
+
`RNImageStitcherPackage.ensureFrameProcessorPluginRegistered`.
|
|
165
|
+
|
|
166
|
+
Plugin contract (identical on both platforms):
|
|
167
|
+
- Args: `path` (string, REQUIRED), `quality` (number 0-100,
|
|
168
|
+
default 75, clamped `[1, 100]`)
|
|
169
|
+
- Returns: `{ ok: true, path, width, height }` OR
|
|
170
|
+
`{ ok: false, error: "..." }`
|
|
171
|
+
|
|
172
|
+
Hosts can call this directly from their own `useFrameProcessor`
|
|
173
|
+
worklet for custom rate-control logic; most consumers use it
|
|
174
|
+
indirectly via Layer 3.
|
|
175
|
+
|
|
176
|
+
#### Layer 2 — `useThrottledFrameProcessor` hook
|
|
177
|
+
|
|
178
|
+
```tsx
|
|
179
|
+
const fp = useThrottledFrameProcessor(
|
|
180
|
+
(frame) => {
|
|
181
|
+
'worklet';
|
|
182
|
+
// Worklet-native processing at sub-frame-rate
|
|
183
|
+
},
|
|
184
|
+
{ sampleHz: 2 },
|
|
185
|
+
[],
|
|
186
|
+
);
|
|
187
|
+
```
|
|
188
|
+
|
|
189
|
+
Pure TS throttle gate over `useFrameProcessor` (v0.8.0). Worklet
|
|
190
|
+
fires up to `sampleHz` times per second; ticks too close together
|
|
191
|
+
dropped via a monotonic-time `useSharedValue` gate.
|
|
192
|
+
|
|
193
|
+
**Use for**: worklet-native processing — native OCR via
|
|
194
|
+
Vision.framework / ML Kit wrapped as vc Frame Processor plugins,
|
|
195
|
+
TFLite ML inference, LiDAR depth (`frame.arDepth`). Direct
|
|
196
|
+
buffer/pose/depth access in the worklet; bridge small bbox-result
|
|
197
|
+
payloads to JS via `runOnJS`.
|
|
198
|
+
|
|
199
|
+
`sampleHz` clamped to `[0.5, 30]`.
|
|
200
|
+
|
|
201
|
+
#### Layer 3 — `useFrameStream` hook
|
|
202
|
+
|
|
203
|
+
```tsx
|
|
204
|
+
const fp = useFrameStream(
|
|
205
|
+
{ sampleHz: 2, quality: 75 },
|
|
206
|
+
(sample) => {
|
|
207
|
+
// JS-thread callback: sample.jpegPath, sample.pose, sample.timestamp
|
|
208
|
+
setThumbnail(sample.jpegPath);
|
|
209
|
+
},
|
|
210
|
+
);
|
|
211
|
+
```
|
|
212
|
+
|
|
213
|
+
Composes Layer 2 + Layer 1 + `runOnJS` bridge to deliver
|
|
214
|
+
`SampledFrame` objects to a JS-thread handler. Slot-reuse
|
|
215
|
+
strategy bounds disk usage to ~4 stale JPEGs.
|
|
216
|
+
|
|
217
|
+
**Use for**: JS-thread consumers — file-path OCR libraries (RN
|
|
218
|
+
modules wrapping ML Kit), cloud upload, thumbnail preview UI,
|
|
219
|
+
JS-side ML (TF.js, transformers.js).
|
|
220
|
+
|
|
221
|
+
`sampleHz` clamped to `[0.5, 10]`; `quality` clamped `[1, 100]`.
|
|
222
|
+
|
|
223
|
+
#### Types
|
|
224
|
+
|
|
225
|
+
- `SampledFrame` — `{ jpegPath, pose, timestamp, width, height }`
|
|
226
|
+
- `FrameStreamOptions` — `{ sampleHz, quality?, outputDir? }`
|
|
227
|
+
- `ThrottledFrameProcessorOptions` — `{ sampleHz }`
|
|
228
|
+
|
|
229
|
+
All exported from `react-native-image-stitcher`.
|
|
230
|
+
|
|
231
|
+
### Documentation
|
|
232
|
+
|
|
233
|
+
- `docs/frame-access-tiers.md` — new comprehensive reference for
|
|
234
|
+
all four host-facing hooks (`useKeyframeStream`,
|
|
235
|
+
`useThrottledFrameProcessor`, `useFrameStream`,
|
|
236
|
+
`useFrameProcessor`) with decision flow, cost envelope, use-case
|
|
237
|
+
mapping, AR vs non-AR mode tradeoff.
|
|
238
|
+
|
|
239
|
+
### Example app
|
|
240
|
+
|
|
241
|
+
`example/App.tsx` now mounts `useFrameStream` at 2 Hz with a
|
|
242
|
+
visible thumbnail overlay (bottom-right corner) — visual proof of
|
|
243
|
+
the Layer 1 + 2 + 3 pipeline working end-to-end on both iPhone
|
|
244
|
+
(60 Hz AR) and Galaxy A35 (30 Hz AR).
|
|
245
|
+
|
|
246
|
+
### Compatibility
|
|
247
|
+
|
|
248
|
+
- Strict additive over v0.8.0. No host changes required.
|
|
249
|
+
- Works in both AR and non-AR modes via v0.8.0's unified
|
|
250
|
+
`useFrameProcessor`.
|
|
251
|
+
- New hooks return `useFrameProcessor`-shape objects compatible
|
|
252
|
+
with `<Camera frameProcessor={...}>` (Phase 5 from v0.8.0).
|
|
253
|
+
|
|
254
|
+
### Known limitations (v0.9.0 — addressed in v0.11.0)
|
|
255
|
+
|
|
256
|
+
- **Layer 3 `useFrameStream` in AR mode**: the Layer 1
|
|
257
|
+
`save_frame_as_jpeg` vc Frame Processor plugin expects a vision-
|
|
258
|
+
camera `Frame` with `.buffer = CMSampleBufferRef`. In AR mode
|
|
259
|
+
the worklet receives a `StitcherFrameHostObject` (v0.8.0
|
|
260
|
+
Phase 4b's JSI host object) without `.buffer` — the plugin call
|
|
261
|
+
returns `{ok: false, ...}` and `useFrameStream` silently skips
|
|
262
|
+
the sample. Hosts needing per-frame native processing in AR
|
|
263
|
+
mode should use **Layer 2 (`useThrottledFrameProcessor`)** —
|
|
264
|
+
it works in both modes and is the right primitive for the
|
|
265
|
+
worklet-native use cases listed in `docs/frame-access-tiers.md`
|
|
266
|
+
(OCR via Vision/ML Kit, TFLite ML detection, LiDAR depth).
|
|
267
|
+
AR-mode Layer 3 support tracked in v0.11.0's plan
|
|
268
|
+
(`docs/plans/2026-05-27-v0.11.0-non-ar-composition.md`) —
|
|
269
|
+
bundled with `useStitcherWorklet` since both extend the
|
|
270
|
+
`__stitcherProxy` host-function infrastructure.
|
|
271
|
+
- **Layer 3 `useFrameStream` in non-AR mode**: wiring the host's
|
|
272
|
+
frameProcessor through `<Camera>` displaces the lib's
|
|
273
|
+
first-party stitching driver (the documented Phase 5 either-or
|
|
274
|
+
constraint from v0.8.0). Non-AR panorama capture won't produce
|
|
275
|
+
stitched output while a host frameProcessor is wired. Tracked
|
|
276
|
+
in v0.11.0 (`useStitcherWorklet` composition).
|
|
277
|
+
|
|
278
|
+
### Notes
|
|
279
|
+
|
|
280
|
+
- Formal SSIM parity gate (Phase 7 of the v0.9.0 plan) was NOT
|
|
281
|
+
run for this release — the layered design doesn't touch
|
|
282
|
+
first-party stitching, so a regression is structurally unlikely.
|
|
283
|
+
Harness still in place from v0.8.0 (`scripts/ssim-compare.py`)
|
|
284
|
+
for any host that wants to run it locally.
|
|
285
|
+
|
|
286
|
+
[0.9.0]: https://github.com/bhargavkanda/react-native-image-stitcher/compare/v0.8.0...v0.9.0
|
|
287
|
+
|
|
19
288
|
## [0.8.0] — 2026-05-27
|
|
20
289
|
|
|
21
290
|
### Added — `useFrameProcessor` hook for host worklets
|
package/android/build.gradle
CHANGED
|
@@ -282,6 +282,16 @@ dependencies {
|
|
|
282
282
|
if (findProject(':react-native-worklets-core') != null) {
|
|
283
283
|
implementation project(':react-native-worklets-core')
|
|
284
284
|
}
|
|
285
|
+
|
|
286
|
+
// v0.10.0 audit #11A — Android JUnit test scaffold. JVM unit
|
|
287
|
+
// tests for pure-Kotlin data wrappers + algorithm helpers that
|
|
288
|
+
// don't need an Android device. Run via
|
|
289
|
+
// `gradlew :react-native-image-stitcher:test`.
|
|
290
|
+
//
|
|
291
|
+
// Kept minimal — only JUnit 4 (matches AGP's default test
|
|
292
|
+
// runner). Hosts that want to add their own android-test
|
|
293
|
+
// dependencies can do so independently.
|
|
294
|
+
testImplementation "junit:junit:4.13.2"
|
|
285
295
|
}
|
|
286
296
|
|
|
287
297
|
// Helper from the React Native gradle convention to read host-app
|
|
@@ -1673,12 +1673,26 @@ class IncrementalStitcher(
|
|
|
1673
1673
|
@ReactMethod
|
|
1674
1674
|
fun refinePanorama(options: ReadableMap, promise: Promise) {
|
|
1675
1675
|
val framePathsArr = options.getArray("framePaths")
|
|
1676
|
+
val requestedCount = framePathsArr?.size() ?: 0
|
|
1677
|
+
// v0.10.0 #15A — emit `validating` at the very top so JS sees
|
|
1678
|
+
// refine activity even when validation fails fast. Frames may
|
|
1679
|
+
// be empty here; report whatever the caller asked for.
|
|
1680
|
+
emitRefineProgress(
|
|
1681
|
+
stage = "validating",
|
|
1682
|
+
fraction = 0.05,
|
|
1683
|
+
frames = requestedCount,
|
|
1684
|
+
errorMessage = null,
|
|
1685
|
+
)
|
|
1676
1686
|
if (framePathsArr == null || framePathsArr.size() < 2) {
|
|
1677
|
-
|
|
1678
|
-
"
|
|
1679
|
-
|
|
1680
|
-
|
|
1687
|
+
val msg = "refinePanorama requires at least 2 framePaths (got " +
|
|
1688
|
+
"$requestedCount)."
|
|
1689
|
+
emitRefineProgress(
|
|
1690
|
+
stage = "error",
|
|
1691
|
+
fraction = 1.0,
|
|
1692
|
+
frames = requestedCount,
|
|
1693
|
+
errorMessage = msg,
|
|
1681
1694
|
)
|
|
1695
|
+
promise.reject("incremental-refine-invalid-input", msg)
|
|
1682
1696
|
return
|
|
1683
1697
|
}
|
|
1684
1698
|
val framePaths = Array(framePathsArr.size()) {
|
|
@@ -1686,10 +1700,14 @@ class IncrementalStitcher(
|
|
|
1686
1700
|
}
|
|
1687
1701
|
val outputPathOpt = options.getString("outputPath")
|
|
1688
1702
|
if (outputPathOpt.isNullOrEmpty()) {
|
|
1689
|
-
|
|
1690
|
-
|
|
1691
|
-
|
|
1703
|
+
val msg = "refinePanorama requires a non-empty outputPath."
|
|
1704
|
+
emitRefineProgress(
|
|
1705
|
+
stage = "error",
|
|
1706
|
+
fraction = 1.0,
|
|
1707
|
+
frames = framePaths.size,
|
|
1708
|
+
errorMessage = msg,
|
|
1692
1709
|
)
|
|
1710
|
+
promise.reject("incremental-refine-invalid-input", msg)
|
|
1693
1711
|
return
|
|
1694
1712
|
}
|
|
1695
1713
|
val outputPath = stripFileScheme(outputPathOpt)
|
|
@@ -1709,16 +1727,26 @@ class IncrementalStitcher(
|
|
|
1709
1727
|
// Pre-flight existence check — same defensive layer iOS has.
|
|
1710
1728
|
for (p in framePaths) {
|
|
1711
1729
|
if (!File(p).exists()) {
|
|
1712
|
-
|
|
1713
|
-
|
|
1714
|
-
|
|
1730
|
+
val msg = "refinePanorama: keyframe missing on disk — $p"
|
|
1731
|
+
emitRefineProgress(
|
|
1732
|
+
stage = "error",
|
|
1733
|
+
fraction = 1.0,
|
|
1734
|
+
frames = framePaths.size,
|
|
1735
|
+
errorMessage = msg,
|
|
1715
1736
|
)
|
|
1737
|
+
promise.reject("incremental-refine-missing-keyframe", msg)
|
|
1716
1738
|
return
|
|
1717
1739
|
}
|
|
1718
1740
|
}
|
|
1719
1741
|
|
|
1720
1742
|
refineScope.launch {
|
|
1721
1743
|
try {
|
|
1744
|
+
emitRefineProgress(
|
|
1745
|
+
stage = "stitching",
|
|
1746
|
+
fraction = 0.1,
|
|
1747
|
+
frames = framePaths.size,
|
|
1748
|
+
errorMessage = null,
|
|
1749
|
+
)
|
|
1722
1750
|
val stitcher = BatchStitcher.bridgeInstance
|
|
1723
1751
|
?: throw IllegalStateException(
|
|
1724
1752
|
"BatchStitcher.bridgeInstance is null — " +
|
|
@@ -1742,6 +1770,18 @@ class IncrementalStitcher(
|
|
|
1742
1770
|
useInscribedRectCrop,
|
|
1743
1771
|
stitchMode = effectiveMode,
|
|
1744
1772
|
)
|
|
1773
|
+
// Stitch returned — BatchStitcher writes the JPEG
|
|
1774
|
+
// synchronously, so "writing" reflects the final
|
|
1775
|
+
// assembly + file I/O cost (which has already been
|
|
1776
|
+
// paid by this point in practice). Emit so JS can
|
|
1777
|
+
// flip its label from "Stitching" to "Writing"
|
|
1778
|
+
// before the done event fires.
|
|
1779
|
+
emitRefineProgress(
|
|
1780
|
+
stage = "writing",
|
|
1781
|
+
fraction = 0.9,
|
|
1782
|
+
frames = framePaths.size,
|
|
1783
|
+
errorMessage = null,
|
|
1784
|
+
)
|
|
1745
1785
|
val framesRequested =
|
|
1746
1786
|
if (dims.size > 2) dims[2] else framePaths.size
|
|
1747
1787
|
val framesIncluded =
|
|
@@ -1757,8 +1797,20 @@ class IncrementalStitcher(
|
|
|
1757
1797
|
putInt("framesDropped", framesRequested - framesIncluded)
|
|
1758
1798
|
putDouble("finalConfidenceThresh", finalConfidenceThresh)
|
|
1759
1799
|
}
|
|
1800
|
+
emitRefineProgress(
|
|
1801
|
+
stage = "done",
|
|
1802
|
+
fraction = 1.0,
|
|
1803
|
+
frames = framePaths.size,
|
|
1804
|
+
errorMessage = null,
|
|
1805
|
+
)
|
|
1760
1806
|
promise.resolve(map)
|
|
1761
1807
|
} catch (t: Throwable) {
|
|
1808
|
+
emitRefineProgress(
|
|
1809
|
+
stage = "error",
|
|
1810
|
+
fraction = 1.0,
|
|
1811
|
+
frames = framePaths.size,
|
|
1812
|
+
errorMessage = t.message ?: t.javaClass.simpleName,
|
|
1813
|
+
)
|
|
1762
1814
|
promise.reject("incremental-refine-failed", t.message, t)
|
|
1763
1815
|
}
|
|
1764
1816
|
}
|
|
@@ -1883,6 +1935,59 @@ class IncrementalStitcher(
|
|
|
1883
1935
|
emitState(state)
|
|
1884
1936
|
}
|
|
1885
1937
|
|
|
1938
|
+
/**
|
|
1939
|
+
* v0.10.0 #15A — emit a refine-pipeline phase update on the same
|
|
1940
|
+
* `IncrementalStateUpdate` channel that carries `isRefining` /
|
|
1941
|
+
* `refinedPanoramaPath`. Five `stage` values fire across the
|
|
1942
|
+
* lifetime of one `refinePanorama` call:
|
|
1943
|
+
*
|
|
1944
|
+
* - "validating" (fraction 0.05) — synchronous input checks
|
|
1945
|
+
* - "stitching" (fraction 0.10) — start of the OpenCV stitch
|
|
1946
|
+
* - "writing" (fraction 0.90) — stitch returned, JPEG written
|
|
1947
|
+
* - "done" (fraction 1.00) — promise about to resolve
|
|
1948
|
+
* - "error" (fraction 1.00) — failure path (errorMessage
|
|
1949
|
+
* is non-null)
|
|
1950
|
+
*
|
|
1951
|
+
* Coarse on purpose: OpenCV's Stitcher doesn't expose stage-by-
|
|
1952
|
+
* stage callbacks, so the 0.10 → 0.90 jump is one opaque step.
|
|
1953
|
+
* JS uses `stage` for the UI label and `fraction` for the spinner.
|
|
1954
|
+
*
|
|
1955
|
+
* iOS sibling: IncrementalStitcher.swift::emitRefineProgress.
|
|
1956
|
+
* Field names + stage strings are kept identical so the JS
|
|
1957
|
+
* subscriber in src/stitching/incremental.ts doesn't branch on
|
|
1958
|
+
* platform.
|
|
1959
|
+
*/
|
|
1960
|
+
private fun emitRefineProgress(
|
|
1961
|
+
stage: String,
|
|
1962
|
+
fraction: Double,
|
|
1963
|
+
frames: Int?,
|
|
1964
|
+
errorMessage: String?,
|
|
1965
|
+
) {
|
|
1966
|
+
val state = Arguments.createMap().apply {
|
|
1967
|
+
putNull("panoramaPath")
|
|
1968
|
+
putInt("width", 0)
|
|
1969
|
+
putInt("height", 0)
|
|
1970
|
+
putInt("acceptedCount", 0)
|
|
1971
|
+
putInt("outcome", 0) // AcceptedHigh
|
|
1972
|
+
putDouble("confidence", 1.0)
|
|
1973
|
+
putDouble("overlapPercent", -1.0)
|
|
1974
|
+
putInt("processingMs", 0)
|
|
1975
|
+
putBoolean("isLandscape", false)
|
|
1976
|
+
putInt("paintedExtent", 0)
|
|
1977
|
+
putInt("panExtent", 0)
|
|
1978
|
+
putInt("keyframeMax", 0)
|
|
1979
|
+
putString("refineStage", stage)
|
|
1980
|
+
putDouble("refineProgress", fraction)
|
|
1981
|
+
if (frames != null) {
|
|
1982
|
+
putInt("refineFrames", frames)
|
|
1983
|
+
}
|
|
1984
|
+
if (errorMessage != null) {
|
|
1985
|
+
putString("refineError", errorMessage)
|
|
1986
|
+
}
|
|
1987
|
+
}
|
|
1988
|
+
emitState(state)
|
|
1989
|
+
}
|
|
1990
|
+
|
|
1886
1991
|
/**
|
|
1887
1992
|
* 2026-05-16 — given the live panorama path, derive a sibling
|
|
1888
1993
|
* path for the refined output. Same algorithm iOS uses:
|
|
@@ -55,19 +55,33 @@ class RNImageStitcherPackage : ReactPackage {
|
|
|
55
55
|
) { proxy, options ->
|
|
56
56
|
CvFlowGateFrameProcessor(proxy, options)
|
|
57
57
|
}
|
|
58
|
+
// v0.9.0 Layer 1 — register `save_frame_as_jpeg`
|
|
59
|
+
// alongside the cv_flow_gate plugin. Same lifecycle,
|
|
60
|
+
// same defensive error handling (the outer try/catch
|
|
61
|
+
// covers both registrations). Either both register
|
|
62
|
+
// or neither does — if vc isn't on the classpath,
|
|
63
|
+
// both calls are skipped together.
|
|
64
|
+
FrameProcessorPluginRegistry.addFrameProcessorPlugin(
|
|
65
|
+
SaveFrameAsJpegPlugin.PLUGIN_NAME,
|
|
66
|
+
) { proxy, options ->
|
|
67
|
+
SaveFrameAsJpegPlugin(proxy, options)
|
|
68
|
+
}
|
|
58
69
|
fpPluginRegistered = true
|
|
59
70
|
} catch (e: NoClassDefFoundError) {
|
|
60
71
|
android.util.Log.i(
|
|
61
72
|
"RNImageStitcherPackage",
|
|
62
73
|
"vision-camera FrameProcessorPluginRegistry not on classpath — "
|
|
63
|
-
+ "skipping cv_flow_gate_process_frame
|
|
64
|
-
+ "(host app doesn't appear to use
|
|
74
|
+
+ "skipping cv_flow_gate_process_frame + save_frame_as_jpeg "
|
|
75
|
+
+ "plugin registration (host app doesn't appear to use "
|
|
76
|
+
+ "Frame Processors).",
|
|
65
77
|
)
|
|
66
78
|
fpPluginRegistered = true // don't retry every package init
|
|
67
79
|
} catch (e: Throwable) {
|
|
68
80
|
android.util.Log.w(
|
|
69
81
|
"RNImageStitcherPackage",
|
|
70
|
-
"Failed to register
|
|
82
|
+
"Failed to register Frame Processor plugins "
|
|
83
|
+
+ "(cv_flow_gate_process_frame / save_frame_as_jpeg): "
|
|
84
|
+
+ e.message,
|
|
71
85
|
)
|
|
72
86
|
fpPluginRegistered = true
|
|
73
87
|
}
|
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
// SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
package io.imagestitcher.rn
|
|
3
|
+
|
|
4
|
+
import android.graphics.ImageFormat
|
|
5
|
+
import android.media.Image
|
|
6
|
+
import android.util.Log
|
|
7
|
+
import androidx.annotation.Keep
|
|
8
|
+
import com.facebook.proguard.annotations.DoNotStrip
|
|
9
|
+
import com.mrousavy.camera.frameprocessors.Frame
|
|
10
|
+
import com.mrousavy.camera.frameprocessors.FrameProcessorPlugin
|
|
11
|
+
import com.mrousavy.camera.frameprocessors.VisionCameraProxy
|
|
12
|
+
import io.imagestitcher.rn.ar.YuvImageConverter
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* v0.9.0 Layer 1 — Android vc Frame Processor plugin that JPEG-
|
|
16
|
+
* encodes the supplied frame to a host-supplied path. Mirror of
|
|
17
|
+
* iOS' `SaveFrameAsJpegPlugin.mm`.
|
|
18
|
+
*
|
|
19
|
+
* Plugin name (must match iOS): `save_frame_as_jpeg`.
|
|
20
|
+
*
|
|
21
|
+
* ## Wrapping the existing encoder
|
|
22
|
+
*
|
|
23
|
+
* The lib already encodes JPEGs from NV21 bytes via
|
|
24
|
+
* `YuvImageConverter.encodeJpegFromNV21` — that's the path
|
|
25
|
+
* `RNSARCameraView.kt`'s keyframe-accept callback uses (line 589,
|
|
26
|
+
* `onAccept = { targetPath -> ... encodeJpegFromNV21(...) }`).
|
|
27
|
+
* This plugin reuses that exact encoder so:
|
|
28
|
+
* - JPEG output is byte-equivalent to the keyframe-accept output
|
|
29
|
+
* (same encoder, same quality knob)
|
|
30
|
+
* - No new encoder maintenance burden
|
|
31
|
+
*
|
|
32
|
+
* vision-camera's `Frame.image` is an `android.media.Image` in
|
|
33
|
+
* `YUV_420_888` format (the camera's native). We pass it through
|
|
34
|
+
* `YuvImageConverter.packNV21(image)` to extract the dense NV21
|
|
35
|
+
* byte array + dims, then `encodeJpegFromNV21(packed, file, q, rot)`
|
|
36
|
+
* does the JPEG write.
|
|
37
|
+
*
|
|
38
|
+
* ## Plugin contract (matches iOS surface exactly)
|
|
39
|
+
*
|
|
40
|
+
* Arguments dict:
|
|
41
|
+
* - `path` (string, REQUIRED): absolute output path.
|
|
42
|
+
* - `quality` (number, optional): 0-100 JPEG quality. Default 75.
|
|
43
|
+
* Clamped to [1, 100].
|
|
44
|
+
*
|
|
45
|
+
* Returns:
|
|
46
|
+
* - On success: `{ "ok" => true, "path" => ..., "width" => ...,
|
|
47
|
+
* "height" => ... }`
|
|
48
|
+
* - On failure: `{ "ok" => false, "error" => "..." }`
|
|
49
|
+
*
|
|
50
|
+
* Errors surfaced via the result map (not thrown) — host worklets
|
|
51
|
+
* can branch on `result.ok` without try/catch. Same convention
|
|
52
|
+
* as iOS.
|
|
53
|
+
*
|
|
54
|
+
* ## Lifetime / threading
|
|
55
|
+
*
|
|
56
|
+
* The supplied `Frame` (and its `Image`) is valid only for the
|
|
57
|
+
* duration of this callback — vision-camera closes the underlying
|
|
58
|
+
* `Image` on return. All Image access (NV21 pack + JPEG encode)
|
|
59
|
+
* happens synchronously inside `callback()`.
|
|
60
|
+
*
|
|
61
|
+
* ## Format restriction
|
|
62
|
+
*
|
|
63
|
+
* Only `YUV_420_888` input is supported (vc Android's standard).
|
|
64
|
+
* Anything else returns `{ ok: false, error: "unsupported format" }`.
|
|
65
|
+
* No format conversion fallback — that would mask bugs in the host
|
|
66
|
+
* camera config.
|
|
67
|
+
*
|
|
68
|
+
* ## Registration
|
|
69
|
+
*
|
|
70
|
+
* Registered in `RNImageStitcherPackage.kt`'s companion-object
|
|
71
|
+
* `ensureFrameProcessorPluginRegistered()`, alongside
|
|
72
|
+
* `cv_flow_gate_process_frame`. Same defensive
|
|
73
|
+
* NoClassDefFoundError handling — if vc isn't on the host's
|
|
74
|
+
* classpath, registration is silently skipped (and the plugin's
|
|
75
|
+
* `init { … }` calls below never happen).
|
|
76
|
+
*/
|
|
77
|
+
@DoNotStrip
|
|
78
|
+
@Keep
|
|
79
|
+
class SaveFrameAsJpegPlugin(
|
|
80
|
+
@Suppress("UNUSED_PARAMETER") proxy: VisionCameraProxy,
|
|
81
|
+
@Suppress("UNUSED_PARAMETER") options: Map<String, Any>?,
|
|
82
|
+
) : FrameProcessorPlugin() {
|
|
83
|
+
|
|
84
|
+
override fun callback(frame: Frame, params: Map<String, Any>?): Any {
|
|
85
|
+
val path = params?.get("path") as? String
|
|
86
|
+
?: return mapOf(
|
|
87
|
+
"ok" to false,
|
|
88
|
+
"error" to "missing required `path` argument",
|
|
89
|
+
)
|
|
90
|
+
val rawQuality = (params["quality"] as? Number)?.toInt() ?: 75
|
|
91
|
+
val quality = rawQuality.coerceIn(1, 100)
|
|
92
|
+
|
|
93
|
+
// Frame may throw if vc already released it.
|
|
94
|
+
val image: Image = try {
|
|
95
|
+
frame.image
|
|
96
|
+
} catch (e: Throwable) {
|
|
97
|
+
return mapOf(
|
|
98
|
+
"ok" to false,
|
|
99
|
+
"error" to "frame invalid: ${e.message}",
|
|
100
|
+
)
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
if (image.format != ImageFormat.YUV_420_888) {
|
|
104
|
+
return mapOf(
|
|
105
|
+
"ok" to false,
|
|
106
|
+
"error" to "unsupported format ${image.format} (need YUV_420_888)",
|
|
107
|
+
)
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
// Pack NV21 — same call site RNSARCameraView uses.
|
|
111
|
+
val packed = YuvImageConverter.packNV21(image)
|
|
112
|
+
?: return mapOf(
|
|
113
|
+
"ok" to false,
|
|
114
|
+
"error" to "YuvImageConverter.packNV21 returned null",
|
|
115
|
+
)
|
|
116
|
+
|
|
117
|
+
// Reuse the lib's existing JPEG encoder. Rotation is 0 here
|
|
118
|
+
// (the host's frame is in camera-native orientation; if they
|
|
119
|
+
// want display orientation they can pass an `orientation`
|
|
120
|
+
// arg in a future version — for v0.9.0 the worklet emits
|
|
121
|
+
// raw-camera-oriented JPEGs, matching the keyframe-accept
|
|
122
|
+
// pipeline's behaviour).
|
|
123
|
+
//
|
|
124
|
+
// Signature: `encodeJpegFromNV21(packed, outputPath: String,
|
|
125
|
+
// jpegQuality: Int, displayRotation: Int): String?` — returns
|
|
126
|
+
// the written path on success, null on failure.
|
|
127
|
+
val encodedPath: String? = try {
|
|
128
|
+
YuvImageConverter.encodeJpegFromNV21(
|
|
129
|
+
packed,
|
|
130
|
+
path,
|
|
131
|
+
jpegQuality = quality,
|
|
132
|
+
displayRotation = 0,
|
|
133
|
+
)
|
|
134
|
+
} catch (e: Throwable) {
|
|
135
|
+
return mapOf(
|
|
136
|
+
"ok" to false,
|
|
137
|
+
"error" to "encodeJpegFromNV21 threw: ${e.message}",
|
|
138
|
+
)
|
|
139
|
+
}
|
|
140
|
+
if (encodedPath == null) {
|
|
141
|
+
return mapOf(
|
|
142
|
+
"ok" to false,
|
|
143
|
+
"error" to "encodeJpegFromNV21 returned null",
|
|
144
|
+
)
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
return mapOf(
|
|
148
|
+
"ok" to true,
|
|
149
|
+
"path" to encodedPath,
|
|
150
|
+
"width" to packed.width,
|
|
151
|
+
"height" to packed.height,
|
|
152
|
+
)
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
companion object {
|
|
156
|
+
private const val TAG = "SaveFrameAsJpegPlugin"
|
|
157
|
+
|
|
158
|
+
/// Plugin name; MUST match iOS + the JS-side
|
|
159
|
+
/// `initFrameProcessorPlugin('save_frame_as_jpeg')` call.
|
|
160
|
+
const val PLUGIN_NAME = "save_frame_as_jpeg"
|
|
161
|
+
}
|
|
162
|
+
}
|
|
@@ -78,4 +78,14 @@ void StitcherWorkletRegistry::_resetForTests() {
|
|
|
78
78
|
_nextId = 0;
|
|
79
79
|
}
|
|
80
80
|
|
|
81
|
+
std::string StitcherWorkletRegistry::_installEntryForTests(
|
|
82
|
+
std::shared_ptr<RNWorklet::WorkletInvoker> invoker) {
|
|
83
|
+
std::lock_guard<std::mutex> lock(_mutex);
|
|
84
|
+
std::ostringstream idStream;
|
|
85
|
+
idStream << "host-" << _nextId++;
|
|
86
|
+
std::string id = idStream.str();
|
|
87
|
+
_entries.push_back({id, std::move(invoker)});
|
|
88
|
+
return id;
|
|
89
|
+
}
|
|
90
|
+
|
|
81
91
|
} // namespace retailens
|