react-native-image-stitcher 0.3.0 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +220 -0
- package/android/src/main/java/io/imagestitcher/rn/RNSARCameraView.kt +137 -124
- package/android/src/main/java/io/imagestitcher/rn/ar/YuvImageConverter.kt +212 -119
- package/dist/camera/Camera.js +70 -58
- package/dist/camera/PanoramaSettings.d.ts +478 -0
- package/dist/camera/PanoramaSettings.js +120 -0
- package/dist/camera/PanoramaSettingsBridge.d.ts +84 -0
- package/dist/camera/PanoramaSettingsBridge.js +208 -0
- package/dist/camera/PanoramaSettingsModal.d.ts +50 -299
- package/dist/camera/PanoramaSettingsModal.js +189 -354
- package/dist/camera/buildPanoramaInitialSettings.d.ts +70 -0
- package/dist/camera/buildPanoramaInitialSettings.js +97 -0
- package/dist/camera/lowMemDevice.d.ts +24 -0
- package/dist/camera/lowMemDevice.js +69 -0
- package/dist/index.d.ts +6 -2
- package/dist/index.js +23 -2
- package/package.json +6 -2
- package/src/camera/Camera.tsx +79 -71
- package/src/camera/PanoramaSettings.ts +605 -0
- package/src/camera/PanoramaSettingsBridge.ts +238 -0
- package/src/camera/PanoramaSettingsModal.tsx +296 -989
- package/src/camera/__tests__/PanoramaSettingsBridge.test.ts +375 -0
- package/src/camera/__tests__/buildPanoramaInitialSettings.test.ts +119 -0
- package/src/camera/__tests__/lowMemDevice.test.ts +52 -0
- package/src/camera/buildPanoramaInitialSettings.ts +139 -0
- package/src/camera/lowMemDevice.ts +71 -0
- package/src/index.ts +42 -3
package/CHANGELOG.md
CHANGED
|
@@ -16,6 +16,226 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
16
16
|
|
|
17
17
|
## [Unreleased]
|
|
18
18
|
|
|
19
|
+
## [0.4.1] — 2026-05-23
|
|
20
|
+
|
|
21
|
+
### Fixed
|
|
22
|
+
- **ARCore Image hold time** (PR #15) — `forwardToIncremental` on
|
|
23
|
+
Android now packs the ARCore `Image` payload synchronously and
|
|
24
|
+
closes the image immediately, rather than holding it across the JNI
|
|
25
|
+
hand-off. Eliminates the "ImageReader: maxImages exceeded" backlog
|
|
26
|
+
that throttled non-keyframe processing on the A35 at high pan
|
|
27
|
+
rates.
|
|
28
|
+
|
|
29
|
+
### Tooling
|
|
30
|
+
- **Example app Metro port pinned to 8082** (cherry-pick from
|
|
31
|
+
`feature/f8-frame-processor-yuv`). Mirrored across
|
|
32
|
+
`example/metro.config.js`, `example/package.json` scripts,
|
|
33
|
+
`example/ios/RNImageStitcherExample/AppDelegate.swift`, and
|
|
34
|
+
`example/android/gradle.properties` to keep CLI builds, IDE
|
|
35
|
+
builds, and Gradle invocations consistent on machines where 8081
|
|
36
|
+
is already taken.
|
|
37
|
+
|
|
38
|
+
### Internal
|
|
39
|
+
- Lockfile sync after the v0.4.0 version bump (Podfile.lock spec
|
|
40
|
+
checksum + npm prune of transitive deps that had drifted from
|
|
41
|
+
branch experimentation). No impact on consumers — example-app
|
|
42
|
+
tooling only.
|
|
43
|
+
|
|
44
|
+
## [0.4.0] — 2026-05-23
|
|
45
|
+
|
|
46
|
+
### v0.4 settings revamp (F10)
|
|
47
|
+
|
|
48
|
+
> [!WARNING]
|
|
49
|
+
> **Breaking type change.** The flat 45-field `PanoramaSettings`
|
|
50
|
+
> interface from v0.3 has been replaced with three engine-discriminated
|
|
51
|
+
> hierarchical types (`PanoramaSettings`, `SlitscanSettings`,
|
|
52
|
+
> `HybridSettings`). Consumers passing custom settings literals to
|
|
53
|
+
> `<Camera>` or to a Layer 2 modal must migrate to the new shape; the
|
|
54
|
+
> v0.3 type is deleted, not aliased. The C++ engine wire format is
|
|
55
|
+
> unchanged — only the JS-side type surface moved.
|
|
56
|
+
>
|
|
57
|
+
> **Migration guide:** [`docs/migrations/v0.3-to-v0.4-panorama-settings.md`](docs/migrations/v0.3-to-v0.4-panorama-settings.md)
|
|
58
|
+
> walks through every recipe (default-only hosts, custom-literal
|
|
59
|
+
> hosts, slit-scan / hybrid hosts, storage migration for persisted
|
|
60
|
+
> settings).
|
|
61
|
+
|
|
62
|
+
#### Why
|
|
63
|
+
|
|
64
|
+
The 2026-05-22 audit (entry below in v0.3.0) traced every
|
|
65
|
+
`PanoramaSettings` field's native consumer and proved the flat type
|
|
66
|
+
mixed three engines' (batch-keyframe, slit-scan, hybrid) settings into
|
|
67
|
+
one bag of disjoint subsets. Hosts had no way to know at the type
|
|
68
|
+
level which settings their chosen engine would even read; the modal
|
|
69
|
+
exposed knobs that were silently ignored on the active engine. The
|
|
70
|
+
revamp splits the type along engine boundaries so the types match what
|
|
71
|
+
each engine actually consumes.
|
|
72
|
+
|
|
73
|
+
#### What changed
|
|
74
|
+
|
|
75
|
+
- **New file:** `src/camera/PanoramaSettings.ts` — `CaptureBaseSettings`
|
|
76
|
+
+ three top-level types (`PanoramaSettings`, `SlitscanSettings`,
|
|
77
|
+
`HybridSettings`), each with co-located `DEFAULT_*_SETTINGS`. Sub-trees
|
|
78
|
+
group related knobs: `stitcher` / `frameSelection.flow` (panorama);
|
|
79
|
+
`painting` / `registration.ncc1d` / `registration.ncc2d.emaSmoothing` /
|
|
80
|
+
`registration.ncc2d.panAxisLock` / `plane` / `advanced` (slitscan).
|
|
81
|
+
- **New file:** `src/camera/PanoramaSettingsBridge.ts` — three pure
|
|
82
|
+
adapter functions (`panoramaSettingsToNativeConfig`,
|
|
83
|
+
`slitscanSettingsToNativeConfig`, `hybridSettingsToNativeConfig`)
|
|
84
|
+
that translate the typed JS tree → the flat
|
|
85
|
+
`Record<string, primitive>` the native bridges consume. Handles
|
|
86
|
+
presence-as-enable (`ncc1d` defined ⇒ `enable1dNcc: true` on the
|
|
87
|
+
wire) and source-conditional plane optionals.
|
|
88
|
+
- **New file:** `src/camera/buildPanoramaInitialSettings.ts` — pure
|
|
89
|
+
helper that translates `<Camera>`'s `default*` props into the
|
|
90
|
+
initial `PanoramaSettings` snapshot. Takes the device's low-mem
|
|
91
|
+
classification as an argument so the function stays pure and
|
|
92
|
+
testable.
|
|
93
|
+
- **Rewritten:** `src/camera/PanoramaSettingsModal.tsx` — now consumes
|
|
94
|
+
the new `PanoramaSettings` shape. UI sections mirror the type tree
|
|
95
|
+
(Capture source, Debug, Stitcher accordion, Frame Selection
|
|
96
|
+
accordion with nested Flow tunables). ~600 LOC smaller than v0.3
|
|
97
|
+
because dead slit-scan / hybrid / video-recording fields are gone.
|
|
98
|
+
- **Rewired:** `src/camera/Camera.tsx` — settings state uses the new
|
|
99
|
+
type; `incremental.start({ config })` now passes
|
|
100
|
+
`panoramaSettingsToNativeConfig(settings)` instead of an inline flat
|
|
101
|
+
dict. IMU translation gate reads
|
|
102
|
+
`settings.frameSelection.flow?.maxTranslationCm`. Debug overlay
|
|
103
|
+
reads `settings.frameSelection.mode` + `settings.stitcher.stitchMode`.
|
|
104
|
+
- **Updated:** `src/index.ts` — exports the new types + adapters; drops
|
|
105
|
+
the deleted v0.3 type.
|
|
106
|
+
- **Test infra:** added `jest` + `ts-jest` + `@types/jest` devDeps; new
|
|
107
|
+
`jest.config.js`, `tsconfig.test.json`, `tsconfig.build.json` (the
|
|
108
|
+
latter excludes `__tests__/` from the shipped `dist/`). 19 tests
|
|
109
|
+
across two suites cover the bridge round-trips, presence-as-enable
|
|
110
|
+
cases, plane-source variants, and prop→settings-tree translation.
|
|
111
|
+
|
|
112
|
+
#### Migration table — v0.3 flat → v0.4 hierarchical
|
|
113
|
+
|
|
114
|
+
For `<Camera>`-consuming hosts (the only public path that took
|
|
115
|
+
`PanoramaSettings` in v0.3):
|
|
116
|
+
|
|
117
|
+
| v0.3 field | v0.4 path |
|
|
118
|
+
|----------------------------------|-------------------------------------------------|
|
|
119
|
+
| `captureSource` | `captureSource` (unchanged) |
|
|
120
|
+
| `debug` | `debug` (unchanged) |
|
|
121
|
+
| `stitchMode` | `stitcher.stitchMode` |
|
|
122
|
+
| `warperType` | `stitcher.warperType` |
|
|
123
|
+
| `blenderType` | `stitcher.blenderType` |
|
|
124
|
+
| `seamFinderType` | `stitcher.seamFinderType` |
|
|
125
|
+
| `enableMaxInscribedRectCrop` | `stitcher.enableMaxInscribedRectCrop` |
|
|
126
|
+
| `frameSelectionMode` | `frameSelection.mode` |
|
|
127
|
+
| `keyframeMaxCount` | `frameSelection.maxKeyframes` |
|
|
128
|
+
| `keyframeOverlapThreshold` | `frameSelection.overlapThreshold` |
|
|
129
|
+
| `flowNoveltyPercentile` | `frameSelection.flow.noveltyPercentile` |
|
|
130
|
+
| `flowEvalEveryNFrames` | `frameSelection.flow.evalEveryNFrames` |
|
|
131
|
+
| `flowMaxTranslationCm` | `frameSelection.flow.maxTranslationCm` |
|
|
132
|
+
| `flowMaxCorners` | `frameSelection.flow.maxCorners` |
|
|
133
|
+
| `flowQualityLevel` | `frameSelection.flow.qualityLevel` |
|
|
134
|
+
| `flowMinDistance` | `frameSelection.flow.minDistance` |
|
|
135
|
+
|
|
136
|
+
#### Deleted from the public type surface
|
|
137
|
+
|
|
138
|
+
These fields were consumed only by slit-scan or hybrid engines (or
|
|
139
|
+
not consumed at all per the audit) and were dead surface on
|
|
140
|
+
`<Camera>`'s batch-keyframe path:
|
|
141
|
+
|
|
142
|
+
- `incrementalEngine` — `<Camera>` always uses `batch-keyframe`; the
|
|
143
|
+
knob never reached this component. Hosts that want slit-scan or
|
|
144
|
+
hybrid build their own capture flow on `incremental.start()` and
|
|
145
|
+
pass `SlitscanSettings` / `HybridSettings` instead.
|
|
146
|
+
- `useARPreview` — superseded by `captureSource` ('ar' / 'non-ar').
|
|
147
|
+
- `useDetectedPlane` — superseded by `SlitscanSettings.plane.source`.
|
|
148
|
+
- `planeSource`, `virtualPlaneDepthMeters`, `arkitPlaneAlignmentThreshold`,
|
|
149
|
+
`planeProjectionStyle` — slit-scan only; on `SlitscanSettings.plane.*`.
|
|
150
|
+
- `slitWidthFraction`, `sliverPosition`, `firstFrameFullFrame`,
|
|
151
|
+
`paintMode` — slit-scan only; on `SlitscanSettings.painting.*`.
|
|
152
|
+
- `acceptGate`, `enableTriangulation`, `enableTriAccumulator`,
|
|
153
|
+
`enable2dNcc`, `enableRansacHomography`, `nccSearchRadius1d`,
|
|
154
|
+
`nccSearchMargin2d`, `nccConfidenceThreshold2d`,
|
|
155
|
+
`enableNcc2dEmaSmoothing`, `ncc2dEmaAlpha`,
|
|
156
|
+
`enableNcc2dPanAxisLock`, `ncc2dCrossAxisLockPx` — slit-scan only;
|
|
157
|
+
on `SlitscanSettings.registration.*`.
|
|
158
|
+
- `hybridProjection` — hybrid only; on `HybridSettings.projection`.
|
|
159
|
+
- `maxRecordingMs`, `framesPerSecond`, `minFrames`, `maxFrames`,
|
|
160
|
+
`quality` — historical video-recording fallback fields with no
|
|
161
|
+
consumer on `<Camera>`'s batch-keyframe path.
|
|
162
|
+
|
|
163
|
+
#### Latent v0.3 bug fixed in passing
|
|
164
|
+
|
|
165
|
+
The v0.3 `<Camera>` accepted a `defaultCaptureSource` prop but the
|
|
166
|
+
internal `buildInitialSettings` function never copied it into
|
|
167
|
+
`settings.captureSource` — only into `arPreference` state. The
|
|
168
|
+
discrepancy meant the wire dict sent to native always reported
|
|
169
|
+
`captureSource: 'ar'` even when the operator's effective source was
|
|
170
|
+
`'non-ar'`, which silently disabled Android's `disableAngularFallback`
|
|
171
|
+
opt-out (audit fix F1). v0.4's `extractPanoramaOverrides` +
|
|
172
|
+
`buildPanoramaInitialSettings` route the prop through correctly.
|
|
173
|
+
Hosts using `defaultCaptureSource="non-ar"` will see native receive
|
|
174
|
+
the matching value for the first time.
|
|
175
|
+
|
|
176
|
+
#### Known limitation — modal Capture-source field vs. AR toggle
|
|
177
|
+
|
|
178
|
+
The on-screen AR toggle button at the bottom of `<Camera>` updates
|
|
179
|
+
`arPreference` state (and through it `effectiveCaptureSource`),
|
|
180
|
+
which decides which preview component mounts. The Capture-source
|
|
181
|
+
segmented control inside the settings modal updates
|
|
182
|
+
`settings.captureSource`, which only affects what's reported to the
|
|
183
|
+
native engine via `panoramaSettingsToNativeConfig` (gates Android's
|
|
184
|
+
angular-fallback opt-out per audit fix F1). These two values can
|
|
185
|
+
drift if the operator toggles the AR button without re-opening
|
|
186
|
+
settings, OR flips the modal field without touching the AR button.
|
|
187
|
+
The on-screen toggle is the canonical UI affordance for the live
|
|
188
|
+
preview path; the modal field is best thought of as a tester escape
|
|
189
|
+
hatch for the wire-format consequence. A future cleanup is to make
|
|
190
|
+
both update the same source of truth — out of scope for v0.4.
|
|
191
|
+
|
|
192
|
+
#### Migration example
|
|
193
|
+
|
|
194
|
+
```ts
|
|
195
|
+
// Before (v0.3)
|
|
196
|
+
const settings: PanoramaSettings = {
|
|
197
|
+
captureSource: 'ar',
|
|
198
|
+
stitchMode: 'auto',
|
|
199
|
+
blenderType: 'multiband',
|
|
200
|
+
flowMaxTranslationCm: 50,
|
|
201
|
+
flowNoveltyPercentile: 0.85,
|
|
202
|
+
keyframeMaxCount: 6,
|
|
203
|
+
frameSelectionMode: 'flow-based',
|
|
204
|
+
// … 40+ more fields
|
|
205
|
+
};
|
|
206
|
+
|
|
207
|
+
// After (v0.4)
|
|
208
|
+
const settings: PanoramaSettings = {
|
|
209
|
+
captureSource: 'ar',
|
|
210
|
+
debug: false,
|
|
211
|
+
stitcher: {
|
|
212
|
+
stitchMode: 'auto',
|
|
213
|
+
warperType: 'plane',
|
|
214
|
+
blenderType: 'multiband',
|
|
215
|
+
seamFinderType: 'graphcut',
|
|
216
|
+
enableMaxInscribedRectCrop: false,
|
|
217
|
+
},
|
|
218
|
+
frameSelection: {
|
|
219
|
+
mode: 'flow-based',
|
|
220
|
+
maxKeyframes: 6,
|
|
221
|
+
overlapThreshold: 0.20,
|
|
222
|
+
flow: {
|
|
223
|
+
noveltyPercentile: 0.85,
|
|
224
|
+
evalEveryNFrames: 5,
|
|
225
|
+
maxTranslationCm: 50,
|
|
226
|
+
maxCorners: 150,
|
|
227
|
+
qualityLevel: 0.01,
|
|
228
|
+
minDistance: 10,
|
|
229
|
+
},
|
|
230
|
+
},
|
|
231
|
+
};
|
|
232
|
+
|
|
233
|
+
// Or just use the default:
|
|
234
|
+
import { DEFAULT_PANORAMA_SETTINGS } from 'react-native-image-stitcher';
|
|
235
|
+
const settings = { ...DEFAULT_PANORAMA_SETTINGS, captureSource: 'non-ar' };
|
|
236
|
+
```
|
|
237
|
+
|
|
238
|
+
|
|
19
239
|
## [0.3.0] — 2026-05-23
|
|
20
240
|
|
|
21
241
|
> [!IMPORTANT]
|
|
@@ -428,133 +428,146 @@ class RNSARCameraView @JvmOverloads constructor(
|
|
|
428
428
|
}
|
|
429
429
|
return
|
|
430
430
|
}
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
//
|
|
460
|
-
//
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
// the same convention the iOS Swift side uses (camera-
|
|
467
|
-
// forward in world space). This keeps the two platforms
|
|
468
|
-
// numerically aligned for the FoV-overlap gate.
|
|
469
|
-
val q = camera.pose.rotationQuaternion // x, y, z, w
|
|
470
|
-
val (yaw, pitch) = quaternionYawPitch(q)
|
|
471
|
-
|
|
472
|
-
// Both FoVs + the full quaternion + intrinsics go to the
|
|
473
|
-
// engine. V6 pose-driven path uses (qx, qy, qz, qw, fx,
|
|
474
|
-
// fy, cx, cy, w, h) to compute the geometrically-exact
|
|
475
|
-
// homography.
|
|
476
|
-
val intrinsics = camera.imageIntrinsics
|
|
477
|
-
val fx = intrinsics.focalLength[0].toDouble()
|
|
478
|
-
val fy = intrinsics.focalLength[1].toDouble()
|
|
479
|
-
val cxIntr = intrinsics.principalPoint[0].toDouble()
|
|
480
|
-
val cyIntr = intrinsics.principalPoint[1].toDouble()
|
|
481
|
-
val w = intrinsics.imageDimensions[0].toDouble()
|
|
482
|
-
val h = intrinsics.imageDimensions[1].toDouble()
|
|
483
|
-
val fovHRad = 2.0 * atan(w / (2.0 * fx))
|
|
484
|
-
val fovVRad = 2.0 * atan(h / (2.0 * fy))
|
|
485
|
-
val fovHDeg = fovHRad * 180.0 / Math.PI
|
|
486
|
-
val fovVDeg = fovVRad * 180.0 / Math.PI
|
|
487
|
-
|
|
488
|
-
// ARCore quaternion comes back in (x, y, z, w) order.
|
|
489
|
-
val qarr = camera.pose.rotationQuaternion
|
|
490
|
-
// P3-F: also extract translation so the KeyframeGate's
|
|
491
|
-
// plane-based ray-projection can compute polygon overlap.
|
|
492
|
-
// Previously these were dropped, forcing the gate into
|
|
493
|
-
// angular-fallback even when a plane was latched.
|
|
494
|
-
val tArr = camera.pose.translation
|
|
495
|
-
|
|
496
|
-
val trackingPoor = camera.trackingState != TrackingState.TRACKING
|
|
497
|
-
val module = IncrementalStitcher.bridgeInstance ?: return
|
|
498
|
-
// 2026-05-15 (B3) — pass current display rotation so the
|
|
499
|
-
// encoded JPEG gets an EXIF orientation tag. Captured into
|
|
500
|
-
// a local val so the lambda below closes over a primitive
|
|
501
|
-
// (avoids re-reading lastDisplayRotation if it shifts
|
|
502
|
-
// between gate-evaluate and lambda invocation).
|
|
503
|
-
val rotationForEncode = if (lastDisplayRotation >= 0)
|
|
504
|
-
lastDisplayRotation else android.view.Surface.ROTATION_0
|
|
505
|
-
// 2026-05-21 (v0.3) — eager JPEG encode is only needed when
|
|
506
|
-
// the engine is in the legacy hybrid/firstwins live-engine
|
|
507
|
-
// mode (which feeds JPEG paths into addFrameAtPath every
|
|
508
|
-
// frame). In batch-keyframe mode (the production Camera
|
|
509
|
-
// component's path), the JPEG is encoded LAZILY inside
|
|
510
|
-
// the onAccept lambda below — only on the ~6 frames per
|
|
511
|
-
// capture that the C++ KeyframeGate actually keeps.
|
|
512
|
-
val legacyJpegPath: String? = if (module.isBatchKeyframeMode) {
|
|
513
|
-
null
|
|
514
|
-
} else {
|
|
515
|
-
YuvImageConverter.encodeToJpeg(
|
|
516
|
-
image,
|
|
517
|
-
tmpJpegFile.absolutePath,
|
|
518
|
-
jpegQuality = 70,
|
|
519
|
-
displayRotation = rotationForEncode,
|
|
520
|
-
)
|
|
431
|
+
|
|
432
|
+
// 2026-05-22 (audit follow-up #19) — minimise ARCore Image
|
|
433
|
+
// hold time.
|
|
434
|
+
//
|
|
435
|
+
// Pre-#19 the Image stayed open through the entire JNI
|
|
436
|
+
// ingest call AND any subsequent JPEG encode (~25 ms in
|
|
437
|
+
// legacy hybrid mode where every frame is encoded eagerly;
|
|
438
|
+
// ~25 ms in batch-keyframe mode for the ~5/60 frames the
|
|
439
|
+
// gate accepts). At 60 Hz ARCore that meant the Image was
|
|
440
|
+
// held 25-30 ms per frame on accepts, starving the Camera2
|
|
441
|
+
// ImageReader's circular buffer pool and risking
|
|
442
|
+
// "BufferQueue has been abandoned" stalls.
|
|
443
|
+
//
|
|
444
|
+
// The fix is mechanical: pack the YUV planes into a
|
|
445
|
+
// JVM-side NV21 byte array (~3 ms), close the Image, and
|
|
446
|
+
// run all subsequent work (JNI ingest + JPEG encode) on
|
|
447
|
+
// the copied bytes. ARCore Camera2 buffer pool stays
|
|
448
|
+
// healthier; latency-sensitive ARCore frames flow through
|
|
449
|
+
// their fixed pool instead of waiting on our JPEG path.
|
|
450
|
+
//
|
|
451
|
+
// The packed.nv21 array's first `width*height` bytes are
|
|
452
|
+
// the Y plane (densely packed, stride = width) — these go
|
|
453
|
+
// to the C++ gate as grayscale. The full array is the
|
|
454
|
+
// input to YuvImageConverter.encodeJpegFromNV21 if the
|
|
455
|
+
// gate accepts (or if we're in legacy eager-encode mode).
|
|
456
|
+
val packed = try {
|
|
457
|
+
YuvImageConverter.packNV21(image)
|
|
458
|
+
} finally {
|
|
459
|
+
// Close ASAP — every microsecond reduces buffer-pool
|
|
460
|
+
// pressure on Camera2. Even if packNV21 returns null
|
|
461
|
+
// (unsupported format), we still need to close.
|
|
462
|
+
try { image.close() } catch (_: Throwable) {}
|
|
463
|
+
} ?: run {
|
|
464
|
+
if (forwardLogTick % 30 == 1) {
|
|
465
|
+
Log.w(TAG, "forwardToIncremental: packNV21 returned null (unexpected format?)")
|
|
521
466
|
}
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
467
|
+
return
|
|
468
|
+
}
|
|
469
|
+
|
|
470
|
+
// Compute yaw + pitch from the ARCore quaternion using
|
|
471
|
+
// the same convention the iOS Swift side uses (camera-
|
|
472
|
+
// forward in world space). This keeps the two platforms
|
|
473
|
+
// numerically aligned for the FoV-overlap gate. `camera`
|
|
474
|
+
// (and `camera.pose`) remain valid after image.close() —
|
|
475
|
+
// they're ARCore Frame metadata, not pixel buffers.
|
|
476
|
+
val q = camera.pose.rotationQuaternion // x, y, z, w
|
|
477
|
+
val (yaw, pitch) = quaternionYawPitch(q)
|
|
478
|
+
|
|
479
|
+
// Both FoVs + the full quaternion + intrinsics go to the
|
|
480
|
+
// engine. V6 pose-driven path uses (qx, qy, qz, qw, fx,
|
|
481
|
+
// fy, cx, cy, w, h) to compute the geometrically-exact
|
|
482
|
+
// homography.
|
|
483
|
+
val intrinsics = camera.imageIntrinsics
|
|
484
|
+
val fx = intrinsics.focalLength[0].toDouble()
|
|
485
|
+
val fy = intrinsics.focalLength[1].toDouble()
|
|
486
|
+
val cxIntr = intrinsics.principalPoint[0].toDouble()
|
|
487
|
+
val cyIntr = intrinsics.principalPoint[1].toDouble()
|
|
488
|
+
val w = intrinsics.imageDimensions[0].toDouble()
|
|
489
|
+
val h = intrinsics.imageDimensions[1].toDouble()
|
|
490
|
+
val fovHRad = 2.0 * atan(w / (2.0 * fx))
|
|
491
|
+
val fovVRad = 2.0 * atan(h / (2.0 * fy))
|
|
492
|
+
val fovHDeg = fovHRad * 180.0 / Math.PI
|
|
493
|
+
val fovVDeg = fovVRad * 180.0 / Math.PI
|
|
494
|
+
|
|
495
|
+
// ARCore quaternion comes back in (x, y, z, w) order.
|
|
496
|
+
val qarr = camera.pose.rotationQuaternion
|
|
497
|
+
// P3-F: also extract translation so the KeyframeGate's
|
|
498
|
+
// plane-based ray-projection can compute polygon overlap.
|
|
499
|
+
// Previously these were dropped, forcing the gate into
|
|
500
|
+
// angular-fallback even when a plane was latched.
|
|
501
|
+
val tArr = camera.pose.translation
|
|
502
|
+
|
|
503
|
+
val trackingPoor = camera.trackingState != TrackingState.TRACKING
|
|
504
|
+
val module = IncrementalStitcher.bridgeInstance ?: return
|
|
505
|
+
// 2026-05-15 (B3) — pass current display rotation so the
|
|
506
|
+
// encoded JPEG gets an EXIF orientation tag. Captured into
|
|
507
|
+
// a local val so the lambda below closes over a primitive
|
|
508
|
+
// (avoids re-reading lastDisplayRotation if it shifts
|
|
509
|
+
// between gate-evaluate and lambda invocation).
|
|
510
|
+
val rotationForEncode = if (lastDisplayRotation >= 0)
|
|
511
|
+
lastDisplayRotation else android.view.Surface.ROTATION_0
|
|
512
|
+
|
|
513
|
+
// 2026-05-21 (v0.3) — eager JPEG encode is only needed when
|
|
514
|
+
// the engine is in the legacy hybrid/firstwins live-engine
|
|
515
|
+
// mode (which feeds JPEG paths into addFrameAtPath every
|
|
516
|
+
// frame). In batch-keyframe mode (the production Camera
|
|
517
|
+
// component's path), the JPEG is encoded LAZILY inside
|
|
518
|
+
// the onAccept lambda below — only on the ~6 frames per
|
|
519
|
+
// capture that the C++ KeyframeGate actually keeps.
|
|
520
|
+
//
|
|
521
|
+
// 2026-05-22 (#19) — the encode now reads from the already-
|
|
522
|
+
// packed NV21 bytes (`packed`), NOT from the live Image
|
|
523
|
+
// (which has been closed above). Same output, no Image
|
|
524
|
+
// hold time.
|
|
525
|
+
val legacyJpegPath: String? = if (module.isBatchKeyframeMode) {
|
|
526
|
+
null
|
|
527
|
+
} else {
|
|
528
|
+
YuvImageConverter.encodeJpegFromNV21(
|
|
529
|
+
packed,
|
|
530
|
+
tmpJpegFile.absolutePath,
|
|
531
|
+
jpegQuality = 70,
|
|
532
|
+
displayRotation = rotationForEncode,
|
|
554
533
|
)
|
|
555
|
-
} finally {
|
|
556
|
-
image.close()
|
|
557
534
|
}
|
|
535
|
+
module.ingestFromARCameraView(
|
|
536
|
+
tx = tArr[0].toDouble(),
|
|
537
|
+
ty = tArr[1].toDouble(),
|
|
538
|
+
tz = tArr[2].toDouble(),
|
|
539
|
+
qx = qarr[0].toDouble(), qy = qarr[1].toDouble(),
|
|
540
|
+
qz = qarr[2].toDouble(), qw = qarr[3].toDouble(),
|
|
541
|
+
fx = fx, fy = fy, cx = cxIntr, cy = cyIntr,
|
|
542
|
+
imageWidth = intrinsics.imageDimensions[0],
|
|
543
|
+
imageHeight = intrinsics.imageDimensions[1],
|
|
544
|
+
yaw = yaw, pitch = pitch,
|
|
545
|
+
fovHorizDegrees = fovHDeg, fovVertDegrees = fovVDeg,
|
|
546
|
+
trackingPoor = trackingPoor,
|
|
547
|
+
// The Y plane lives at packed.nv21[0 .. width*height).
|
|
548
|
+
// C++ keyframe_gate reads `height * stride` bytes and
|
|
549
|
+
// ignores anything past that, so passing the full NV21
|
|
550
|
+
// array with `grayStride = width` reads exactly the Y
|
|
551
|
+
// plane (UV bytes at the tail are not touched).
|
|
552
|
+
grayData = packed.nv21,
|
|
553
|
+
grayWidth = packed.width,
|
|
554
|
+
grayHeight = packed.height,
|
|
555
|
+
grayStride = packed.width,
|
|
556
|
+
legacyJpegPath = legacyJpegPath,
|
|
557
|
+
onAccept = { targetPath ->
|
|
558
|
+
// Lazy JPEG encode. Runs ONLY if the C++ KeyframeGate
|
|
559
|
+
// accepted the frame. Encodes from the pre-packed
|
|
560
|
+
// NV21 bytes — the ARCore Image has been closed since
|
|
561
|
+
// ~25 ms ago (right after packNV21), so no
|
|
562
|
+
// Image-hold cost on this slow path.
|
|
563
|
+
YuvImageConverter.encodeJpegFromNV21(
|
|
564
|
+
packed,
|
|
565
|
+
targetPath,
|
|
566
|
+
jpegQuality = 70,
|
|
567
|
+
displayRotation = rotationForEncode,
|
|
568
|
+
) != null
|
|
569
|
+
},
|
|
570
|
+
)
|
|
558
571
|
}
|
|
559
572
|
|
|
560
573
|
private fun applyDisplayGeometry() {
|