react-native-image-stitcher 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +96 -0
- package/LICENSE +201 -0
- package/NOTICE +21 -0
- package/README.md +189 -0
- package/RNImageStitcher.podspec +76 -0
- package/android/build.gradle +224 -0
- package/android/src/main/AndroidManifest.xml +3 -0
- package/android/src/main/cpp/CMakeLists.txt +124 -0
- package/android/src/main/cpp/image_stitcher_jni.cpp +145 -0
- package/android/src/main/cpp/keyframe_gate_jni.cpp +204 -0
- package/android/src/main/java/io/imagestitcher/rn/BatchStitcher.kt +426 -0
- package/android/src/main/java/io/imagestitcher/rn/IncrementalFirstwinsEngine.kt +960 -0
- package/android/src/main/java/io/imagestitcher/rn/IncrementalStitcher.kt +2371 -0
- package/android/src/main/java/io/imagestitcher/rn/KeyframeGate.kt +256 -0
- package/android/src/main/java/io/imagestitcher/rn/QualityChecker.kt +167 -0
- package/android/src/main/java/io/imagestitcher/rn/RNImageStitcherPackage.kt +39 -0
- package/android/src/main/java/io/imagestitcher/rn/RNSARCameraView.kt +558 -0
- package/android/src/main/java/io/imagestitcher/rn/RNSARCameraViewManager.kt +35 -0
- package/android/src/main/java/io/imagestitcher/rn/RNSARSession.kt +784 -0
- package/android/src/main/java/io/imagestitcher/rn/ar/BackgroundRenderer.kt +176 -0
- package/android/src/main/java/io/imagestitcher/rn/ar/ShaderUtil.kt +67 -0
- package/android/src/main/java/io/imagestitcher/rn/ar/YuvImageConverter.kt +201 -0
- package/cpp/ar_frame_pose.h +63 -0
- package/cpp/keyframe_gate.cpp +927 -0
- package/cpp/keyframe_gate.hpp +240 -0
- package/cpp/stitcher.cpp +2207 -0
- package/cpp/stitcher.hpp +275 -0
- package/dist/ar/useARSession.d.ts +102 -0
- package/dist/ar/useARSession.js +133 -0
- package/dist/camera/ARCameraView.d.ts +93 -0
- package/dist/camera/ARCameraView.js +170 -0
- package/dist/camera/Camera.d.ts +134 -0
- package/dist/camera/Camera.js +688 -0
- package/dist/camera/CameraShutter.d.ts +80 -0
- package/dist/camera/CameraShutter.js +237 -0
- package/dist/camera/CameraView.d.ts +65 -0
- package/dist/camera/CameraView.js +117 -0
- package/dist/camera/CaptureControlsBar.d.ts +87 -0
- package/dist/camera/CaptureControlsBar.js +82 -0
- package/dist/camera/CaptureHeader.d.ts +62 -0
- package/dist/camera/CaptureHeader.js +81 -0
- package/dist/camera/CapturePreview.d.ts +70 -0
- package/dist/camera/CapturePreview.js +188 -0
- package/dist/camera/CaptureStatusOverlay.d.ts +75 -0
- package/dist/camera/CaptureStatusOverlay.js +326 -0
- package/dist/camera/CaptureThumbnailStrip.d.ts +87 -0
- package/dist/camera/CaptureThumbnailStrip.js +177 -0
- package/dist/camera/IncrementalPanGuide.d.ts +83 -0
- package/dist/camera/IncrementalPanGuide.js +267 -0
- package/dist/camera/PanoramaBandOverlay.d.ts +107 -0
- package/dist/camera/PanoramaBandOverlay.js +399 -0
- package/dist/camera/PanoramaConfirmModal.d.ts +57 -0
- package/dist/camera/PanoramaConfirmModal.js +128 -0
- package/dist/camera/PanoramaGuidance.d.ts +79 -0
- package/dist/camera/PanoramaGuidance.js +246 -0
- package/dist/camera/PanoramaSettingsModal.d.ts +311 -0
- package/dist/camera/PanoramaSettingsModal.js +611 -0
- package/dist/camera/ViewportCropOverlay.d.ts +46 -0
- package/dist/camera/ViewportCropOverlay.js +67 -0
- package/dist/camera/useCapture.d.ts +111 -0
- package/dist/camera/useCapture.js +160 -0
- package/dist/camera/useDeviceOrientation.d.ts +48 -0
- package/dist/camera/useDeviceOrientation.js +131 -0
- package/dist/camera/useVideoCapture.d.ts +79 -0
- package/dist/camera/useVideoCapture.js +151 -0
- package/dist/index.d.ts +26 -0
- package/dist/index.js +39 -0
- package/dist/quality/normaliseOrientation.d.ts +36 -0
- package/dist/quality/normaliseOrientation.js +62 -0
- package/dist/quality/runQualityCheck.d.ts +41 -0
- package/dist/quality/runQualityCheck.js +98 -0
- package/dist/sensors/useIMUTranslationGate.d.ts +70 -0
- package/dist/sensors/useIMUTranslationGate.js +235 -0
- package/dist/stitching/IncrementalStitcherView.d.ts +41 -0
- package/dist/stitching/IncrementalStitcherView.js +157 -0
- package/dist/stitching/incremental.d.ts +930 -0
- package/dist/stitching/incremental.js +133 -0
- package/dist/stitching/stitchFrames.d.ts +55 -0
- package/dist/stitching/stitchFrames.js +56 -0
- package/dist/stitching/stitchVideo.d.ts +119 -0
- package/dist/stitching/stitchVideo.js +57 -0
- package/dist/stitching/useIncrementalJSDriver.d.ts +74 -0
- package/dist/stitching/useIncrementalJSDriver.js +199 -0
- package/dist/stitching/useIncrementalStitcher.d.ts +58 -0
- package/dist/stitching/useIncrementalStitcher.js +172 -0
- package/dist/types.d.ts +58 -0
- package/dist/types.js +15 -0
- package/ios/Package.swift +72 -0
- package/ios/Sources/RNImageStitcher/ARCameraViewManager.m +33 -0
- package/ios/Sources/RNImageStitcher/ARCameraViewManager.swift +40 -0
- package/ios/Sources/RNImageStitcher/ARSessionBridge.m +55 -0
- package/ios/Sources/RNImageStitcher/ARSessionBridge.swift +149 -0
- package/ios/Sources/RNImageStitcher/IncrementalStitcher.swift +2727 -0
- package/ios/Sources/RNImageStitcher/IncrementalStitcherBridge.m +85 -0
- package/ios/Sources/RNImageStitcher/IncrementalStitcherBridge.swift +625 -0
- package/ios/Sources/RNImageStitcher/KeyframeGate.swift +328 -0
- package/ios/Sources/RNImageStitcher/KeyframeGateBridge.h +141 -0
- package/ios/Sources/RNImageStitcher/KeyframeGateBridge.mm +278 -0
- package/ios/Sources/RNImageStitcher/OpenCVIncrementalStitcher.h +473 -0
- package/ios/Sources/RNImageStitcher/OpenCVIncrementalStitcher.mm +1326 -0
- package/ios/Sources/RNImageStitcher/OpenCVKeyframeCollector.h +97 -0
- package/ios/Sources/RNImageStitcher/OpenCVKeyframeCollector.mm +296 -0
- package/ios/Sources/RNImageStitcher/OpenCVSlitScanStitcher.h +103 -0
- package/ios/Sources/RNImageStitcher/OpenCVSlitScanStitcher.mm +3285 -0
- package/ios/Sources/RNImageStitcher/OpenCVStitcher.h +238 -0
- package/ios/Sources/RNImageStitcher/OpenCVStitcher.mm +1880 -0
- package/ios/Sources/RNImageStitcher/QualityChecker.swift +252 -0
- package/ios/Sources/RNImageStitcher/QualityCheckerBridge.m +26 -0
- package/ios/Sources/RNImageStitcher/QualityCheckerBridge.swift +72 -0
- package/ios/Sources/RNImageStitcher/RNSARCameraView.swift +114 -0
- package/ios/Sources/RNImageStitcher/RNSARSession.swift +1111 -0
- package/ios/Sources/RNImageStitcher/Stitcher.swift +243 -0
- package/ios/Sources/RNImageStitcher/StitcherBridge.m +28 -0
- package/ios/Sources/RNImageStitcher/StitcherBridge.swift +246 -0
- package/package.json +73 -0
- package/react-native.config.js +34 -0
- package/scripts/opencv-version.txt +1 -0
- package/scripts/postinstall-fetch-binaries.js +286 -0
- package/src/ar/useARSession.ts +210 -0
- package/src/camera/.gitkeep +0 -0
- package/src/camera/ARCameraView.tsx +256 -0
- package/src/camera/Camera.tsx +1053 -0
- package/src/camera/CameraShutter.tsx +292 -0
- package/src/camera/CameraView.tsx +157 -0
- package/src/camera/CaptureControlsBar.tsx +204 -0
- package/src/camera/CaptureHeader.tsx +184 -0
- package/src/camera/CapturePreview.tsx +318 -0
- package/src/camera/CaptureStatusOverlay.tsx +391 -0
- package/src/camera/CaptureThumbnailStrip.tsx +277 -0
- package/src/camera/IncrementalPanGuide.tsx +328 -0
- package/src/camera/PanoramaBandOverlay.tsx +498 -0
- package/src/camera/PanoramaConfirmModal.tsx +206 -0
- package/src/camera/PanoramaGuidance.tsx +327 -0
- package/src/camera/PanoramaSettingsModal.tsx +1357 -0
- package/src/camera/ViewportCropOverlay.tsx +81 -0
- package/src/camera/useCapture.ts +279 -0
- package/src/camera/useDeviceOrientation.ts +140 -0
- package/src/camera/useVideoCapture.ts +236 -0
- package/src/index.ts +53 -0
- package/src/quality/.gitkeep +0 -0
- package/src/quality/normaliseOrientation.ts +79 -0
- package/src/quality/runQualityCheck.ts +131 -0
- package/src/sensors/useIMUTranslationGate.ts +347 -0
- package/src/stitching/.gitkeep +0 -0
- package/src/stitching/IncrementalStitcherView.tsx +198 -0
- package/src/stitching/incremental.ts +1021 -0
- package/src/stitching/stitchFrames.ts +88 -0
- package/src/stitching/stitchVideo.ts +153 -0
- package/src/stitching/useIncrementalJSDriver.ts +273 -0
- package/src/stitching/useIncrementalStitcher.ts +252 -0
- package/src/types.ts +78 -0
|
@@ -0,0 +1,558 @@
|
|
|
1
|
+
// SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
package io.imagestitcher.rn
|
|
3
|
+
|
|
4
|
+
import android.content.Context
|
|
5
|
+
import android.opengl.GLES20
|
|
6
|
+
import android.opengl.GLSurfaceView
|
|
7
|
+
import android.os.Handler
|
|
8
|
+
import android.os.Looper
|
|
9
|
+
import android.util.AttributeSet
|
|
10
|
+
import android.util.Log
|
|
11
|
+
import android.view.Surface
|
|
12
|
+
import android.view.WindowManager
|
|
13
|
+
import android.widget.FrameLayout
|
|
14
|
+
import com.google.ar.core.Camera
|
|
15
|
+
import com.google.ar.core.Session
|
|
16
|
+
import com.google.ar.core.TrackingState
|
|
17
|
+
import com.google.ar.core.exceptions.CameraNotAvailableException
|
|
18
|
+
import com.google.ar.core.exceptions.SessionPausedException
|
|
19
|
+
import io.imagestitcher.rn.ar.BackgroundRenderer
|
|
20
|
+
import io.imagestitcher.rn.ar.YuvImageConverter
|
|
21
|
+
import java.io.File
|
|
22
|
+
import java.util.concurrent.atomic.AtomicReference
|
|
23
|
+
import javax.microedition.khronos.egl.EGLConfig
|
|
24
|
+
import javax.microedition.khronos.opengles.GL10
|
|
25
|
+
import kotlin.math.atan
|
|
26
|
+
import kotlin.math.atan2
|
|
27
|
+
import kotlin.math.asin
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* Android twin of `RNSARCameraView.swift` (iOS Phase 4.4).
|
|
31
|
+
*
|
|
32
|
+
* Embeds a `GLSurfaceView` that renders the ARCore camera feed and
|
|
33
|
+
* drives the AR session's per-frame `update()` loop on the GL render
|
|
34
|
+
* thread. When the incremental stitcher is running, each frame's
|
|
35
|
+
* camera image is converted to JPEG and fed into the engine with the
|
|
36
|
+
* matching ARCore pose — full parity with iOS' ARSession path (no
|
|
37
|
+
* gyro fallback needed when this view is mounted).
|
|
38
|
+
*
|
|
39
|
+
* Why a GLSurfaceView and not a TextureView / SurfaceView:
|
|
40
|
+
* ARCore needs a GL_TEXTURE_EXTERNAL_OES texture as its camera
|
|
41
|
+
* sink (Session.setCameraTextureName). Only a GLSurfaceView with
|
|
42
|
+
* EGL14 context gives us the OES extension. The Renderer
|
|
43
|
+
* callback is also where the per-frame Session.update() lives, so
|
|
44
|
+
* the threading model lines up cleanly.
|
|
45
|
+
*
|
|
46
|
+
* Lifecycle:
|
|
47
|
+
* onAttachedToWindow → mark "wants to render", borrow Session
|
|
48
|
+
* from RNSARSession.instance
|
|
49
|
+
* onSurfaceCreated (GL) → create OES texture, build BackgroundRenderer
|
|
50
|
+
* onSurfaceChanged (GL) → notify session of display geometry
|
|
51
|
+
* onDrawFrame (GL) → session.update(); pose → log;
|
|
52
|
+
* if stitcher running: image → JPEG → engine
|
|
53
|
+
* onDetachedFromWindow → pause render thread; do NOT pause Session
|
|
54
|
+
* (other views may still be using it)
|
|
55
|
+
*/
|
|
56
|
+
class RNSARCameraView @JvmOverloads constructor(
|
|
57
|
+
context: Context,
|
|
58
|
+
attrs: AttributeSet? = null,
|
|
59
|
+
defStyle: Int = 0,
|
|
60
|
+
) : FrameLayout(context, attrs, defStyle), GLSurfaceView.Renderer {
|
|
61
|
+
|
|
62
|
+
private val glView: GLSurfaceView = GLSurfaceView(context).also { v ->
|
|
63
|
+
v.preserveEGLContextOnPause = true
|
|
64
|
+
v.setEGLContextClientVersion(2)
|
|
65
|
+
v.setEGLConfigChooser(8, 8, 8, 8, 16, 0)
|
|
66
|
+
v.setRenderer(this)
|
|
67
|
+
v.renderMode = GLSurfaceView.RENDERMODE_CONTINUOUSLY
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
private val backgroundRenderer = BackgroundRenderer()
|
|
71
|
+
private val sessionRef = AtomicReference<Session?>(null)
|
|
72
|
+
private var sessionTextureBound = false
|
|
73
|
+
/// Last known display rotation; consulted on each setDisplayGeometry
|
|
74
|
+
/// call so we can recompute when the user rotates the device.
|
|
75
|
+
private var lastDisplayRotation: Int = -1
|
|
76
|
+
private var surfaceWidth: Int = 0
|
|
77
|
+
private var surfaceHeight: Int = 0
|
|
78
|
+
|
|
79
|
+
/// Tmp directory for the per-frame JPEG file we hand to the
|
|
80
|
+
/// incremental engine. Created lazily and reused across frames
|
|
81
|
+
/// — no per-frame allocation.
|
|
82
|
+
private val tmpJpegFile: File by lazy {
|
|
83
|
+
File(context.cacheDir, "rlis-arframe.jpg")
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
/// Whether to feed the AR session's frames into the incremental
|
|
87
|
+
/// engine. Toggled by IncrementalStitcher.start/stop
|
|
88
|
+
/// via setIncrementalIngestionActive() below.
|
|
89
|
+
@Volatile private var ingestActive: Boolean = false
|
|
90
|
+
|
|
91
|
+
/// Pending takePhoto request, populated by `requestTakePhoto`
|
|
92
|
+
/// from the bridge thread and consumed by the GL render thread
|
|
93
|
+
/// on the next `onDrawFrame` so the latest ARCore frame is
|
|
94
|
+
/// captured. Cleared atomically so concurrent shutter taps
|
|
95
|
+
/// don't double-fire — the second tap's promise rejects the
|
|
96
|
+
/// older request before replacing it.
|
|
97
|
+
internal data class TakePhotoRequest(
|
|
98
|
+
val outputPath: String,
|
|
99
|
+
val quality: Int,
|
|
100
|
+
val promise: com.facebook.react.bridge.Promise,
|
|
101
|
+
)
|
|
102
|
+
private val pendingTakePhoto =
|
|
103
|
+
AtomicReference<TakePhotoRequest?>(null)
|
|
104
|
+
|
|
105
|
+
/// Called from the bridge (RNSARSession.takePhoto @ReactMethod).
|
|
106
|
+
/// Stores a request that will be fulfilled on the next render
|
|
107
|
+
/// tick. If another request is already queued, that one is
|
|
108
|
+
/// rejected (the JS layer should serialise its own calls).
|
|
109
|
+
internal fun requestTakePhoto(
|
|
110
|
+
outputPath: String,
|
|
111
|
+
quality: Int,
|
|
112
|
+
promise: com.facebook.react.bridge.Promise,
|
|
113
|
+
) {
|
|
114
|
+
val req = TakePhotoRequest(outputPath, quality, promise)
|
|
115
|
+
val previous = pendingTakePhoto.getAndSet(req)
|
|
116
|
+
previous?.promise?.reject(
|
|
117
|
+
"ar-photo-superseded",
|
|
118
|
+
"takePhoto: superseded by a newer call before the frame was captured.",
|
|
119
|
+
)
|
|
120
|
+
// Wake the render loop in case it's idle.
|
|
121
|
+
glView.requestRender()
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
init {
|
|
125
|
+
addView(
|
|
126
|
+
glView,
|
|
127
|
+
LayoutParams(LayoutParams.MATCH_PARENT, LayoutParams.MATCH_PARENT),
|
|
128
|
+
)
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
override fun onAttachedToWindow() {
|
|
132
|
+
super.onAttachedToWindow()
|
|
133
|
+
Log.i(TAG, "onAttachedToWindow: requesting AR session start (iOS-parity didMoveToWindow)")
|
|
134
|
+
// iOS parity (didMoveToWindow): ensure the singleton AR
|
|
135
|
+
// session is running BEFORE we try to borrow it for
|
|
136
|
+
// rendering. Previously the view only borrowed an existing
|
|
137
|
+
// session — if nothing else had started one yet, the
|
|
138
|
+
// GLSurfaceView would stay at its cleared-black state
|
|
139
|
+
// forever and the user would see a black camera preview.
|
|
140
|
+
//
|
|
141
|
+
// startForView() is idempotent (no-op if a session is
|
|
142
|
+
// already running) and silently logs failures rather than
|
|
143
|
+
// throwing — if it returns false the view falls through to
|
|
144
|
+
// the borrow logic below, which then renders empty. Worst-
|
|
145
|
+
// case the user navigates away + back to retry.
|
|
146
|
+
RNSARSession.instance?.startForView()
|
|
147
|
+
|
|
148
|
+
glView.onResume()
|
|
149
|
+
// Try to borrow the session from the running RNSARSession.
|
|
150
|
+
val session = RNSARSession.instance?.getSessionForView()
|
|
151
|
+
if (session != null) {
|
|
152
|
+
sessionRef.set(session)
|
|
153
|
+
// ARCore's `Session.resume()` must be called on the main
|
|
154
|
+
// thread — startForView() above already resumed a freshly-
|
|
155
|
+
// created session, but if we got here with a pre-existing
|
|
156
|
+
// paused session (e.g. another ARCameraView's onDetached
|
|
157
|
+
// ran and paused, then this view re-mounted) we resume
|
|
158
|
+
// again here. Idempotent: Session.resume() is a no-op
|
|
159
|
+
// if the session is already resumed.
|
|
160
|
+
try {
|
|
161
|
+
session.resume()
|
|
162
|
+
} catch (e: CameraNotAvailableException) {
|
|
163
|
+
Log.w(TAG, "session.resume on attach: $e")
|
|
164
|
+
}
|
|
165
|
+
} else {
|
|
166
|
+
Log.w(
|
|
167
|
+
TAG,
|
|
168
|
+
"onAttachedToWindow: session is still null after startForView; " +
|
|
169
|
+
"preview will stay black until the view re-mounts " +
|
|
170
|
+
"(possible reasons: no Activity, ARCore install in progress, " +
|
|
171
|
+
"device unsupported — see RNSARSession logs)",
|
|
172
|
+
)
|
|
173
|
+
}
|
|
174
|
+
RNSARSession.instance?.bindCameraView(this)
|
|
175
|
+
IncrementalStitcher.bridgeInstance?.bindArCameraView(this)
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
override fun onDetachedFromWindow() {
|
|
179
|
+
super.onDetachedFromWindow()
|
|
180
|
+
Log.i(TAG, "onDetachedFromWindow: requesting AR session stop (iOS-parity didMoveToWindow)")
|
|
181
|
+
// Pause the GL thread so we stop drawing frames.
|
|
182
|
+
glView.onPause()
|
|
183
|
+
sessionTextureBound = false
|
|
184
|
+
IncrementalStitcher.bridgeInstance?.unbindArCameraView(this)
|
|
185
|
+
RNSARSession.instance?.unbindCameraView(this)
|
|
186
|
+
// iOS parity (didMoveToWindow else-branch): stop the session
|
|
187
|
+
// so the hardware camera is freed for vision-camera or other
|
|
188
|
+
// consumers when the user navigates away. Updated from the
|
|
189
|
+
// previous "do NOT pause the session" comment, which assumed
|
|
190
|
+
// the bridge module's start/stop owned lifecycle exclusively.
|
|
191
|
+
// With startForView()/stopForView(), the view is now the
|
|
192
|
+
// primary lifecycle owner for the auto-mounted case (the
|
|
193
|
+
// most common path: AuditCaptureScreen mounts the view, which
|
|
194
|
+
// starts the session; navigates away, which stops it).
|
|
195
|
+
// The JS-facing `start(promise)` / `stop(promise)` continue
|
|
196
|
+
// to work for hosts that prefer explicit control — the
|
|
197
|
+
// refs/state are shared.
|
|
198
|
+
RNSARSession.instance?.stopForView()
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
/// Called by IncrementalStitcher.start/stop. When true,
|
|
202
|
+
/// each ARCore frame's camera image is encoded to JPEG + handed
|
|
203
|
+
/// to the engine; when false, the per-frame work skips ingestion
|
|
204
|
+
/// (the camera feed continues to render either way).
|
|
205
|
+
fun setIncrementalIngestionActive(active: Boolean) {
|
|
206
|
+
// P3-G diagnostic — surfaces whether the camera view ever
|
|
207
|
+
// got "engage the ingestion path" command from the stitcher.
|
|
208
|
+
// Common failure mode: stitcher.start() ran, but
|
|
209
|
+
// arCameraViewRef was null (view not bound yet) → this never
|
|
210
|
+
// fires → forwardToIncremental never runs → 0 frames.
|
|
211
|
+
Log.i(TAG, "setIncrementalIngestionActive: $active (prev=$ingestActive)")
|
|
212
|
+
ingestActive = active
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
// ── GLSurfaceView.Renderer ─────────────────────────────────────
|
|
216
|
+
|
|
217
|
+
override fun onSurfaceCreated(gl: GL10?, config: EGLConfig?) {
|
|
218
|
+
GLES20.glClearColor(0f, 0f, 0f, 1f)
|
|
219
|
+
backgroundRenderer.createOnGlThread()
|
|
220
|
+
sessionTextureBound = false
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
override fun onSurfaceChanged(gl: GL10?, width: Int, height: Int) {
|
|
224
|
+
GLES20.glViewport(0, 0, width, height)
|
|
225
|
+
surfaceWidth = width
|
|
226
|
+
surfaceHeight = height
|
|
227
|
+
applyDisplayGeometry()
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
override fun onDrawFrame(gl: GL10?) {
|
|
231
|
+
GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT or GLES20.GL_DEPTH_BUFFER_BIT)
|
|
232
|
+
|
|
233
|
+
val session = sessionRef.get() ?: run {
|
|
234
|
+
// Session not yet attached (start() hasn't run, or
|
|
235
|
+
// the bridge module instance was rebuilt). Try once
|
|
236
|
+
// more in case the bridge resolved it after onAttach.
|
|
237
|
+
val late = RNSARSession.instance?.getSessionForView()
|
|
238
|
+
if (late != null) sessionRef.set(late)
|
|
239
|
+
return
|
|
240
|
+
}
|
|
241
|
+
if (!sessionTextureBound) {
|
|
242
|
+
backgroundRenderer.bindToSession(session)
|
|
243
|
+
sessionTextureBound = true
|
|
244
|
+
// Ensure ARCore knows the surface geometry.
|
|
245
|
+
applyDisplayGeometry()
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
val frame = try {
|
|
249
|
+
session.update()
|
|
250
|
+
} catch (e: SessionPausedException) {
|
|
251
|
+
return // session paused — wait for resume
|
|
252
|
+
} catch (t: Throwable) {
|
|
253
|
+
Log.w(TAG, "session.update failed: ${t.message}")
|
|
254
|
+
return
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
// Draw the camera background regardless of tracking state —
|
|
258
|
+
// gives the user something to look at while AR initialises.
|
|
259
|
+
backgroundRenderer.draw(frame)
|
|
260
|
+
|
|
261
|
+
val camera: Camera = frame.camera
|
|
262
|
+
|
|
263
|
+
// ── V15.0e — vertical plane detection (iOS parity) ──────────
|
|
264
|
+
// Run this each frame so the JS 2 Hz getARPlaneStatus poll
|
|
265
|
+
// sees a live answer without the user having to take any
|
|
266
|
+
// action. iOS ARKit re-runs evaluation internally on each
|
|
267
|
+
// ARSessionDelegate didUpdate callback; we mirror by polling
|
|
268
|
+
// every ARCore frame. Cost: ~10-20 us per frame at idle (no
|
|
269
|
+
// planes), ~50-100 us when iterating a handful of tracking
|
|
270
|
+
// planes — negligible against the 16ms frame budget.
|
|
271
|
+
val pose = camera.pose
|
|
272
|
+
// ARCore Pose convention: zAxis is the world-space direction
|
|
273
|
+
// of the local Z axis. Camera looks down -Z (OpenGL
|
|
274
|
+
// convention), so cameraForward = -zAxis. ARCore 1.45's
|
|
275
|
+
// Pose.getZAxis() takes no args and returns a new FloatArray.
|
|
276
|
+
val zAxis = pose.zAxis
|
|
277
|
+
val cameraForwardWorld = floatArrayOf(-zAxis[0], -zAxis[1], -zAxis[2])
|
|
278
|
+
val cameraPosWorld = floatArrayOf(pose.tx(), pose.ty(), pose.tz())
|
|
279
|
+
RNSARSession.instance?.evaluatePlanesForFrame(
|
|
280
|
+
cameraForwardWorld,
|
|
281
|
+
cameraPosWorld,
|
|
282
|
+
)
|
|
283
|
+
|
|
284
|
+
// Push pose into the AR session log. Mirrors iOS' delegate
|
|
285
|
+
// path; the existing RNSARFramePose / appendPose
|
|
286
|
+
// contract was already in place for Phase 4.
|
|
287
|
+
appendPose(camera, frame.timestamp)
|
|
288
|
+
|
|
289
|
+
// Forward to the incremental stitcher if engaged.
|
|
290
|
+
if (ingestActive) {
|
|
291
|
+
forwardToIncremental(frame, camera)
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
// takePhoto consumer — runs on EVERY render tick (not just
|
|
295
|
+
// when ingest is active), since the host calls takePhoto in
|
|
296
|
+
// photo mode where ingest is off. No-op when no request is
|
|
297
|
+
// pending; cheap atomic CAS on the hot path.
|
|
298
|
+
pendingTakePhoto.getAndSet(null)?.let { req ->
|
|
299
|
+
fulfilTakePhoto(frame, req)
|
|
300
|
+
}
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
/// Capture the current ARCore frame to JPEG and resolve / reject
|
|
304
|
+
/// `req.promise`. Runs on the GL render thread, called from
|
|
305
|
+
/// `onDrawFrame` after the frame has been obtained via
|
|
306
|
+
/// `session.update()`. Mirrors iOS' `RNSARSession.takePhoto`
|
|
307
|
+
/// resolution shape: `{ path, width, height, isMirrored,
|
|
308
|
+
/// isRawPhoto }` so JS code is platform-agnostic.
|
|
309
|
+
private fun fulfilTakePhoto(
|
|
310
|
+
frame: com.google.ar.core.Frame,
|
|
311
|
+
req: TakePhotoRequest,
|
|
312
|
+
) {
|
|
313
|
+
val image = try {
|
|
314
|
+
frame.acquireCameraImage()
|
|
315
|
+
} catch (t: Throwable) {
|
|
316
|
+
req.promise.reject(
|
|
317
|
+
"ar-photo-no-frame",
|
|
318
|
+
"takePhoto: acquireCameraImage failed: ${t.message}",
|
|
319
|
+
)
|
|
320
|
+
return
|
|
321
|
+
}
|
|
322
|
+
val width = image.width
|
|
323
|
+
val height = image.height
|
|
324
|
+
try {
|
|
325
|
+
val written = YuvImageConverter.encodeToJpeg(
|
|
326
|
+
image,
|
|
327
|
+
req.outputPath,
|
|
328
|
+
jpegQuality = req.quality.coerceIn(1, 100),
|
|
329
|
+
displayRotation = if (lastDisplayRotation >= 0)
|
|
330
|
+
lastDisplayRotation
|
|
331
|
+
else
|
|
332
|
+
Surface.ROTATION_0,
|
|
333
|
+
)
|
|
334
|
+
if (written == null) {
|
|
335
|
+
req.promise.reject(
|
|
336
|
+
"ar-photo-encode-failed",
|
|
337
|
+
"takePhoto: YuvImageConverter.encodeToJpeg returned null.",
|
|
338
|
+
)
|
|
339
|
+
return
|
|
340
|
+
}
|
|
341
|
+
val result = com.facebook.react.bridge.Arguments.createMap().apply {
|
|
342
|
+
putString("path", written)
|
|
343
|
+
putInt("width", width)
|
|
344
|
+
putInt("height", height)
|
|
345
|
+
putBoolean("isMirrored", false)
|
|
346
|
+
putBoolean("isRawPhoto", false)
|
|
347
|
+
}
|
|
348
|
+
req.promise.resolve(result)
|
|
349
|
+
} catch (t: Throwable) {
|
|
350
|
+
req.promise.reject(
|
|
351
|
+
"ar-photo-failed",
|
|
352
|
+
"takePhoto: unexpected error: ${t.message}",
|
|
353
|
+
t,
|
|
354
|
+
)
|
|
355
|
+
} finally {
|
|
356
|
+
// Image must always be closed or ARCore will starve.
|
|
357
|
+
try { image.close() } catch (_: Throwable) {}
|
|
358
|
+
}
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
private fun appendPose(camera: Camera, timestampNs: Long) {
|
|
362
|
+
val pose = camera.pose
|
|
363
|
+
val translation = pose.translation
|
|
364
|
+
val rotation = pose.rotationQuaternion // x, y, z, w
|
|
365
|
+
val intrinsics = camera.imageIntrinsics
|
|
366
|
+
val focal = intrinsics.focalLength
|
|
367
|
+
val principal = intrinsics.principalPoint
|
|
368
|
+
val dims = intrinsics.imageDimensions
|
|
369
|
+
|
|
370
|
+
val tracking = when (camera.trackingState) {
|
|
371
|
+
TrackingState.TRACKING -> RNSARSession.TRACKING_TRACKING
|
|
372
|
+
TrackingState.PAUSED -> RNSARSession.TRACKING_LIMITED
|
|
373
|
+
TrackingState.STOPPED -> RNSARSession.TRACKING_NOT_AVAILABLE
|
|
374
|
+
else -> RNSARSession.TRACKING_NOT_AVAILABLE
|
|
375
|
+
}
|
|
376
|
+
|
|
377
|
+
val framePose = RNSARFramePose(
|
|
378
|
+
tx = translation[0].toDouble(),
|
|
379
|
+
ty = translation[1].toDouble(),
|
|
380
|
+
tz = translation[2].toDouble(),
|
|
381
|
+
qx = rotation[0].toDouble(),
|
|
382
|
+
qy = rotation[1].toDouble(),
|
|
383
|
+
qz = rotation[2].toDouble(),
|
|
384
|
+
qw = rotation[3].toDouble(),
|
|
385
|
+
fx = focal[0].toDouble(),
|
|
386
|
+
fy = focal[1].toDouble(),
|
|
387
|
+
cx = principal[0].toDouble(),
|
|
388
|
+
cy = principal[1].toDouble(),
|
|
389
|
+
imageWidth = dims[0],
|
|
390
|
+
imageHeight = dims[1],
|
|
391
|
+
timestampMs = timestampNs / 1_000_000.0,
|
|
392
|
+
trackingState = tracking,
|
|
393
|
+
)
|
|
394
|
+
RNSARSession.instance?.appendPose(framePose)
|
|
395
|
+
RNSARSession.instance?.updateTrackingState(camera.trackingState)
|
|
396
|
+
}
|
|
397
|
+
|
|
398
|
+
/// P3-G diagnostic — rate-limit the per-frame log so we can see
|
|
399
|
+
/// at-a-glance whether forwardToIncremental is even running, vs
|
|
400
|
+
/// being short-circuited at the `if (ingestActive)` guard in
|
|
401
|
+
/// onDrawFrame.
|
|
402
|
+
private var forwardLogTick: Int = 0
|
|
403
|
+
|
|
404
|
+
private fun forwardToIncremental(
|
|
405
|
+
frame: com.google.ar.core.Frame,
|
|
406
|
+
camera: Camera,
|
|
407
|
+
) {
|
|
408
|
+
if (forwardLogTick++ % 30 == 0) {
|
|
409
|
+
Log.i(TAG, "forwardToIncremental: ingestActive=$ingestActive trackingState=${camera.trackingState}")
|
|
410
|
+
}
|
|
411
|
+
// Acquire the camera image. Each call may throw
|
|
412
|
+
// NotYetAvailableException for the first ~1-2 frames before
|
|
413
|
+
// ARCore catches up — silently skip those.
|
|
414
|
+
val image = try {
|
|
415
|
+
frame.acquireCameraImage()
|
|
416
|
+
} catch (t: Throwable) {
|
|
417
|
+
if (forwardLogTick % 30 == 1) {
|
|
418
|
+
Log.w(TAG, "forwardToIncremental: acquireCameraImage failed: ${t.message}")
|
|
419
|
+
}
|
|
420
|
+
return
|
|
421
|
+
}
|
|
422
|
+
try {
|
|
423
|
+
val written = YuvImageConverter.encodeToJpeg(
|
|
424
|
+
image,
|
|
425
|
+
tmpJpegFile.absolutePath,
|
|
426
|
+
jpegQuality = 70,
|
|
427
|
+
// 2026-05-15 (B3) — pass current display rotation so
|
|
428
|
+
// the encoded JPEG gets an EXIF orientation tag.
|
|
429
|
+
// Without this, the live thumbnail strip shows
|
|
430
|
+
// sideways pictures when the device is held in
|
|
431
|
+
// portrait (sensor pixels are landscape by default).
|
|
432
|
+
// lastDisplayRotation is updated by the
|
|
433
|
+
// updateDisplayRotation() helper called from
|
|
434
|
+
// didMoveToWindow / the ARCore Session.setDisplayGeometry
|
|
435
|
+
// hook (see line ~410).
|
|
436
|
+
displayRotation = if (lastDisplayRotation >= 0)
|
|
437
|
+
lastDisplayRotation else android.view.Surface.ROTATION_0,
|
|
438
|
+
) ?: return
|
|
439
|
+
|
|
440
|
+
// Compute yaw + pitch from the ARCore quaternion using
|
|
441
|
+
// the same convention the iOS Swift side uses (camera-
|
|
442
|
+
// forward in world space). This keeps the two platforms
|
|
443
|
+
// numerically aligned for the FoV-overlap gate.
|
|
444
|
+
val q = camera.pose.rotationQuaternion // x, y, z, w
|
|
445
|
+
val (yaw, pitch) = quaternionYawPitch(q)
|
|
446
|
+
|
|
447
|
+
// Both FoVs + the full quaternion + intrinsics go to the
|
|
448
|
+
// engine. V6 pose-driven path uses (qx, qy, qz, qw, fx,
|
|
449
|
+
// fy, cx, cy, w, h) to compute the geometrically-exact
|
|
450
|
+
// homography.
|
|
451
|
+
val intrinsics = camera.imageIntrinsics
|
|
452
|
+
val fx = intrinsics.focalLength[0].toDouble()
|
|
453
|
+
val fy = intrinsics.focalLength[1].toDouble()
|
|
454
|
+
val cxIntr = intrinsics.principalPoint[0].toDouble()
|
|
455
|
+
val cyIntr = intrinsics.principalPoint[1].toDouble()
|
|
456
|
+
val w = intrinsics.imageDimensions[0].toDouble()
|
|
457
|
+
val h = intrinsics.imageDimensions[1].toDouble()
|
|
458
|
+
val fovHRad = 2.0 * atan(w / (2.0 * fx))
|
|
459
|
+
val fovVRad = 2.0 * atan(h / (2.0 * fy))
|
|
460
|
+
val fovHDeg = fovHRad * 180.0 / Math.PI
|
|
461
|
+
val fovVDeg = fovVRad * 180.0 / Math.PI
|
|
462
|
+
|
|
463
|
+
// ARCore quaternion comes back in (x, y, z, w) order.
|
|
464
|
+
val qarr = camera.pose.rotationQuaternion
|
|
465
|
+
// P3-F: also extract translation so the KeyframeGate's
|
|
466
|
+
// plane-based ray-projection can compute polygon overlap.
|
|
467
|
+
// Previously these were dropped, forcing the gate into
|
|
468
|
+
// angular-fallback even when a plane was latched.
|
|
469
|
+
val tArr = camera.pose.translation
|
|
470
|
+
|
|
471
|
+
val trackingPoor = camera.trackingState != TrackingState.TRACKING
|
|
472
|
+
postFrameToEngine(
|
|
473
|
+
path = written,
|
|
474
|
+
tx = tArr[0].toDouble(),
|
|
475
|
+
ty = tArr[1].toDouble(),
|
|
476
|
+
tz = tArr[2].toDouble(),
|
|
477
|
+
qx = qarr[0].toDouble(), qy = qarr[1].toDouble(),
|
|
478
|
+
qz = qarr[2].toDouble(), qw = qarr[3].toDouble(),
|
|
479
|
+
fx = fx, fy = fy, cx = cxIntr, cy = cyIntr,
|
|
480
|
+
imageWidth = intrinsics.imageDimensions[0],
|
|
481
|
+
imageHeight = intrinsics.imageDimensions[1],
|
|
482
|
+
yaw = yaw, pitch = pitch,
|
|
483
|
+
fovHorizDegrees = fovHDeg, fovVertDegrees = fovVDeg,
|
|
484
|
+
trackingPoor = trackingPoor,
|
|
485
|
+
)
|
|
486
|
+
} finally {
|
|
487
|
+
image.close()
|
|
488
|
+
}
|
|
489
|
+
}
|
|
490
|
+
|
|
491
|
+
private fun postFrameToEngine(
|
|
492
|
+
path: String,
|
|
493
|
+
tx: Double, ty: Double, tz: Double,
|
|
494
|
+
qx: Double, qy: Double, qz: Double, qw: Double,
|
|
495
|
+
fx: Double, fy: Double, cx: Double, cy: Double,
|
|
496
|
+
imageWidth: Int, imageHeight: Int,
|
|
497
|
+
yaw: Double,
|
|
498
|
+
pitch: Double,
|
|
499
|
+
fovHorizDegrees: Double,
|
|
500
|
+
fovVertDegrees: Double,
|
|
501
|
+
trackingPoor: Boolean,
|
|
502
|
+
) {
|
|
503
|
+
val module = IncrementalStitcher.bridgeInstance ?: return
|
|
504
|
+
module.ingestFromARCameraView(
|
|
505
|
+
path = path,
|
|
506
|
+
tx = tx, ty = ty, tz = tz,
|
|
507
|
+
qx = qx, qy = qy, qz = qz, qw = qw,
|
|
508
|
+
fx = fx, fy = fy, cx = cx, cy = cy,
|
|
509
|
+
imageWidth = imageWidth, imageHeight = imageHeight,
|
|
510
|
+
yaw = yaw,
|
|
511
|
+
pitch = pitch,
|
|
512
|
+
fovHorizDegrees = fovHorizDegrees,
|
|
513
|
+
fovVertDegrees = fovVertDegrees,
|
|
514
|
+
trackingPoor = trackingPoor,
|
|
515
|
+
)
|
|
516
|
+
}
|
|
517
|
+
|
|
518
|
+
private fun applyDisplayGeometry() {
|
|
519
|
+
val session = sessionRef.get() ?: return
|
|
520
|
+
val rotation = (context.getSystemService(Context.WINDOW_SERVICE) as? WindowManager)
|
|
521
|
+
?.defaultDisplay
|
|
522
|
+
?.rotation
|
|
523
|
+
?: Surface.ROTATION_0
|
|
524
|
+
if (rotation != lastDisplayRotation
|
|
525
|
+
|| surfaceWidth > 0 || surfaceHeight > 0
|
|
526
|
+
) {
|
|
527
|
+
session.setDisplayGeometry(rotation, surfaceWidth, surfaceHeight)
|
|
528
|
+
lastDisplayRotation = rotation
|
|
529
|
+
}
|
|
530
|
+
}
|
|
531
|
+
|
|
532
|
+
/**
|
|
533
|
+
* Convert an ARCore quaternion (x, y, z, w) to (yaw, pitch) in
|
|
534
|
+
* radians — same convention as the iOS Swift side: rotate the
|
|
535
|
+
* camera-forward (-Z) vector by the quaternion, then pull yaw
|
|
536
|
+
* (atan2 onto X-Z plane) and pitch (asin of Y component).
|
|
537
|
+
*
|
|
538
|
+
* Closed-form forward = R * (0, 0, -1), where R is the standard
|
|
539
|
+
* quaternion-to-3x3 matrix. Bottom row of R gives `forwardZ`,
|
|
540
|
+
* etc; multiplying (0,0,-1) just negates the third column.
|
|
541
|
+
*/
|
|
542
|
+
private fun quaternionYawPitch(q: FloatArray): Pair<Double, Double> {
|
|
543
|
+
val x = q[0].toDouble()
|
|
544
|
+
val y = q[1].toDouble()
|
|
545
|
+
val z = q[2].toDouble()
|
|
546
|
+
val w = q[3].toDouble()
|
|
547
|
+
val forwardX = -(2.0 * (x * z + w * y))
|
|
548
|
+
val forwardY = -(2.0 * (y * z - w * x))
|
|
549
|
+
val forwardZ = -(1.0 - 2.0 * (x * x + y * y))
|
|
550
|
+
val yaw = atan2(forwardX, -forwardZ)
|
|
551
|
+
val pitch = asin(forwardY.coerceIn(-1.0, 1.0))
|
|
552
|
+
return yaw to pitch
|
|
553
|
+
}
|
|
554
|
+
|
|
555
|
+
companion object {
|
|
556
|
+
private const val TAG = "RNSARCameraView"
|
|
557
|
+
}
|
|
558
|
+
}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
// SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
package io.imagestitcher.rn
|
|
3
|
+
|
|
4
|
+
import com.facebook.react.uimanager.SimpleViewManager
|
|
5
|
+
import com.facebook.react.uimanager.ThemedReactContext
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* RN ViewManager for `RNSARCameraView`.
|
|
9
|
+
*
|
|
10
|
+
* JS side imports it as:
|
|
11
|
+
*
|
|
12
|
+
* import { requireNativeComponent } from 'react-native';
|
|
13
|
+
* const RNARCamera = requireNativeComponent('RNSARCameraView');
|
|
14
|
+
*
|
|
15
|
+
* Or — preferred — uses the SDK's existing `<ARCameraView>` wrapper
|
|
16
|
+
* in `src/camera/ARCameraView.tsx` which auto-selects the native
|
|
17
|
+
* component for iOS vs Android.
|
|
18
|
+
*
|
|
19
|
+
* The view itself is config-free for now (no JS-side props beyond
|
|
20
|
+
* `style`) since lifecycle is driven by mount/unmount + the
|
|
21
|
+
* incremental stitcher's start/finalize methods. Future phases may
|
|
22
|
+
* add props like `enabled` to allow JS-controlled pause/resume of
|
|
23
|
+
* the GL render loop.
|
|
24
|
+
*/
|
|
25
|
+
class RNSARCameraViewManager : SimpleViewManager<RNSARCameraView>() {
|
|
26
|
+
|
|
27
|
+
override fun getName(): String = REACT_CLASS
|
|
28
|
+
|
|
29
|
+
override fun createViewInstance(reactContext: ThemedReactContext): RNSARCameraView =
|
|
30
|
+
RNSARCameraView(reactContext)
|
|
31
|
+
|
|
32
|
+
companion object {
|
|
33
|
+
const val REACT_CLASS = "RNSARCameraView"
|
|
34
|
+
}
|
|
35
|
+
}
|