react-native-image-stitcher 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (151) hide show
  1. package/CHANGELOG.md +96 -0
  2. package/LICENSE +201 -0
  3. package/NOTICE +21 -0
  4. package/README.md +189 -0
  5. package/RNImageStitcher.podspec +76 -0
  6. package/android/build.gradle +224 -0
  7. package/android/src/main/AndroidManifest.xml +3 -0
  8. package/android/src/main/cpp/CMakeLists.txt +124 -0
  9. package/android/src/main/cpp/image_stitcher_jni.cpp +145 -0
  10. package/android/src/main/cpp/keyframe_gate_jni.cpp +204 -0
  11. package/android/src/main/java/io/imagestitcher/rn/BatchStitcher.kt +426 -0
  12. package/android/src/main/java/io/imagestitcher/rn/IncrementalFirstwinsEngine.kt +960 -0
  13. package/android/src/main/java/io/imagestitcher/rn/IncrementalStitcher.kt +2371 -0
  14. package/android/src/main/java/io/imagestitcher/rn/KeyframeGate.kt +256 -0
  15. package/android/src/main/java/io/imagestitcher/rn/QualityChecker.kt +167 -0
  16. package/android/src/main/java/io/imagestitcher/rn/RNImageStitcherPackage.kt +39 -0
  17. package/android/src/main/java/io/imagestitcher/rn/RNSARCameraView.kt +558 -0
  18. package/android/src/main/java/io/imagestitcher/rn/RNSARCameraViewManager.kt +35 -0
  19. package/android/src/main/java/io/imagestitcher/rn/RNSARSession.kt +784 -0
  20. package/android/src/main/java/io/imagestitcher/rn/ar/BackgroundRenderer.kt +176 -0
  21. package/android/src/main/java/io/imagestitcher/rn/ar/ShaderUtil.kt +67 -0
  22. package/android/src/main/java/io/imagestitcher/rn/ar/YuvImageConverter.kt +201 -0
  23. package/cpp/ar_frame_pose.h +63 -0
  24. package/cpp/keyframe_gate.cpp +927 -0
  25. package/cpp/keyframe_gate.hpp +240 -0
  26. package/cpp/stitcher.cpp +2207 -0
  27. package/cpp/stitcher.hpp +275 -0
  28. package/dist/ar/useARSession.d.ts +102 -0
  29. package/dist/ar/useARSession.js +133 -0
  30. package/dist/camera/ARCameraView.d.ts +93 -0
  31. package/dist/camera/ARCameraView.js +170 -0
  32. package/dist/camera/Camera.d.ts +134 -0
  33. package/dist/camera/Camera.js +688 -0
  34. package/dist/camera/CameraShutter.d.ts +80 -0
  35. package/dist/camera/CameraShutter.js +237 -0
  36. package/dist/camera/CameraView.d.ts +65 -0
  37. package/dist/camera/CameraView.js +117 -0
  38. package/dist/camera/CaptureControlsBar.d.ts +87 -0
  39. package/dist/camera/CaptureControlsBar.js +82 -0
  40. package/dist/camera/CaptureHeader.d.ts +62 -0
  41. package/dist/camera/CaptureHeader.js +81 -0
  42. package/dist/camera/CapturePreview.d.ts +70 -0
  43. package/dist/camera/CapturePreview.js +188 -0
  44. package/dist/camera/CaptureStatusOverlay.d.ts +75 -0
  45. package/dist/camera/CaptureStatusOverlay.js +326 -0
  46. package/dist/camera/CaptureThumbnailStrip.d.ts +87 -0
  47. package/dist/camera/CaptureThumbnailStrip.js +177 -0
  48. package/dist/camera/IncrementalPanGuide.d.ts +83 -0
  49. package/dist/camera/IncrementalPanGuide.js +267 -0
  50. package/dist/camera/PanoramaBandOverlay.d.ts +107 -0
  51. package/dist/camera/PanoramaBandOverlay.js +399 -0
  52. package/dist/camera/PanoramaConfirmModal.d.ts +57 -0
  53. package/dist/camera/PanoramaConfirmModal.js +128 -0
  54. package/dist/camera/PanoramaGuidance.d.ts +79 -0
  55. package/dist/camera/PanoramaGuidance.js +246 -0
  56. package/dist/camera/PanoramaSettingsModal.d.ts +311 -0
  57. package/dist/camera/PanoramaSettingsModal.js +611 -0
  58. package/dist/camera/ViewportCropOverlay.d.ts +46 -0
  59. package/dist/camera/ViewportCropOverlay.js +67 -0
  60. package/dist/camera/useCapture.d.ts +111 -0
  61. package/dist/camera/useCapture.js +160 -0
  62. package/dist/camera/useDeviceOrientation.d.ts +48 -0
  63. package/dist/camera/useDeviceOrientation.js +131 -0
  64. package/dist/camera/useVideoCapture.d.ts +79 -0
  65. package/dist/camera/useVideoCapture.js +151 -0
  66. package/dist/index.d.ts +26 -0
  67. package/dist/index.js +39 -0
  68. package/dist/quality/normaliseOrientation.d.ts +36 -0
  69. package/dist/quality/normaliseOrientation.js +62 -0
  70. package/dist/quality/runQualityCheck.d.ts +41 -0
  71. package/dist/quality/runQualityCheck.js +98 -0
  72. package/dist/sensors/useIMUTranslationGate.d.ts +70 -0
  73. package/dist/sensors/useIMUTranslationGate.js +235 -0
  74. package/dist/stitching/IncrementalStitcherView.d.ts +41 -0
  75. package/dist/stitching/IncrementalStitcherView.js +157 -0
  76. package/dist/stitching/incremental.d.ts +930 -0
  77. package/dist/stitching/incremental.js +133 -0
  78. package/dist/stitching/stitchFrames.d.ts +55 -0
  79. package/dist/stitching/stitchFrames.js +56 -0
  80. package/dist/stitching/stitchVideo.d.ts +119 -0
  81. package/dist/stitching/stitchVideo.js +57 -0
  82. package/dist/stitching/useIncrementalJSDriver.d.ts +74 -0
  83. package/dist/stitching/useIncrementalJSDriver.js +199 -0
  84. package/dist/stitching/useIncrementalStitcher.d.ts +58 -0
  85. package/dist/stitching/useIncrementalStitcher.js +172 -0
  86. package/dist/types.d.ts +58 -0
  87. package/dist/types.js +15 -0
  88. package/ios/Package.swift +72 -0
  89. package/ios/Sources/RNImageStitcher/ARCameraViewManager.m +33 -0
  90. package/ios/Sources/RNImageStitcher/ARCameraViewManager.swift +40 -0
  91. package/ios/Sources/RNImageStitcher/ARSessionBridge.m +55 -0
  92. package/ios/Sources/RNImageStitcher/ARSessionBridge.swift +149 -0
  93. package/ios/Sources/RNImageStitcher/IncrementalStitcher.swift +2727 -0
  94. package/ios/Sources/RNImageStitcher/IncrementalStitcherBridge.m +85 -0
  95. package/ios/Sources/RNImageStitcher/IncrementalStitcherBridge.swift +625 -0
  96. package/ios/Sources/RNImageStitcher/KeyframeGate.swift +328 -0
  97. package/ios/Sources/RNImageStitcher/KeyframeGateBridge.h +141 -0
  98. package/ios/Sources/RNImageStitcher/KeyframeGateBridge.mm +278 -0
  99. package/ios/Sources/RNImageStitcher/OpenCVIncrementalStitcher.h +473 -0
  100. package/ios/Sources/RNImageStitcher/OpenCVIncrementalStitcher.mm +1326 -0
  101. package/ios/Sources/RNImageStitcher/OpenCVKeyframeCollector.h +97 -0
  102. package/ios/Sources/RNImageStitcher/OpenCVKeyframeCollector.mm +296 -0
  103. package/ios/Sources/RNImageStitcher/OpenCVSlitScanStitcher.h +103 -0
  104. package/ios/Sources/RNImageStitcher/OpenCVSlitScanStitcher.mm +3285 -0
  105. package/ios/Sources/RNImageStitcher/OpenCVStitcher.h +238 -0
  106. package/ios/Sources/RNImageStitcher/OpenCVStitcher.mm +1880 -0
  107. package/ios/Sources/RNImageStitcher/QualityChecker.swift +252 -0
  108. package/ios/Sources/RNImageStitcher/QualityCheckerBridge.m +26 -0
  109. package/ios/Sources/RNImageStitcher/QualityCheckerBridge.swift +72 -0
  110. package/ios/Sources/RNImageStitcher/RNSARCameraView.swift +114 -0
  111. package/ios/Sources/RNImageStitcher/RNSARSession.swift +1111 -0
  112. package/ios/Sources/RNImageStitcher/Stitcher.swift +243 -0
  113. package/ios/Sources/RNImageStitcher/StitcherBridge.m +28 -0
  114. package/ios/Sources/RNImageStitcher/StitcherBridge.swift +246 -0
  115. package/package.json +73 -0
  116. package/react-native.config.js +34 -0
  117. package/scripts/opencv-version.txt +1 -0
  118. package/scripts/postinstall-fetch-binaries.js +286 -0
  119. package/src/ar/useARSession.ts +210 -0
  120. package/src/camera/.gitkeep +0 -0
  121. package/src/camera/ARCameraView.tsx +256 -0
  122. package/src/camera/Camera.tsx +1053 -0
  123. package/src/camera/CameraShutter.tsx +292 -0
  124. package/src/camera/CameraView.tsx +157 -0
  125. package/src/camera/CaptureControlsBar.tsx +204 -0
  126. package/src/camera/CaptureHeader.tsx +184 -0
  127. package/src/camera/CapturePreview.tsx +318 -0
  128. package/src/camera/CaptureStatusOverlay.tsx +391 -0
  129. package/src/camera/CaptureThumbnailStrip.tsx +277 -0
  130. package/src/camera/IncrementalPanGuide.tsx +328 -0
  131. package/src/camera/PanoramaBandOverlay.tsx +498 -0
  132. package/src/camera/PanoramaConfirmModal.tsx +206 -0
  133. package/src/camera/PanoramaGuidance.tsx +327 -0
  134. package/src/camera/PanoramaSettingsModal.tsx +1357 -0
  135. package/src/camera/ViewportCropOverlay.tsx +81 -0
  136. package/src/camera/useCapture.ts +279 -0
  137. package/src/camera/useDeviceOrientation.ts +140 -0
  138. package/src/camera/useVideoCapture.ts +236 -0
  139. package/src/index.ts +53 -0
  140. package/src/quality/.gitkeep +0 -0
  141. package/src/quality/normaliseOrientation.ts +79 -0
  142. package/src/quality/runQualityCheck.ts +131 -0
  143. package/src/sensors/useIMUTranslationGate.ts +347 -0
  144. package/src/stitching/.gitkeep +0 -0
  145. package/src/stitching/IncrementalStitcherView.tsx +198 -0
  146. package/src/stitching/incremental.ts +1021 -0
  147. package/src/stitching/stitchFrames.ts +88 -0
  148. package/src/stitching/stitchVideo.ts +153 -0
  149. package/src/stitching/useIncrementalJSDriver.ts +273 -0
  150. package/src/stitching/useIncrementalStitcher.ts +252 -0
  151. package/src/types.ts +78 -0
@@ -0,0 +1,558 @@
1
+ // SPDX-License-Identifier: Apache-2.0
2
+ package io.imagestitcher.rn
3
+
4
+ import android.content.Context
5
+ import android.opengl.GLES20
6
+ import android.opengl.GLSurfaceView
7
+ import android.os.Handler
8
+ import android.os.Looper
9
+ import android.util.AttributeSet
10
+ import android.util.Log
11
+ import android.view.Surface
12
+ import android.view.WindowManager
13
+ import android.widget.FrameLayout
14
+ import com.google.ar.core.Camera
15
+ import com.google.ar.core.Session
16
+ import com.google.ar.core.TrackingState
17
+ import com.google.ar.core.exceptions.CameraNotAvailableException
18
+ import com.google.ar.core.exceptions.SessionPausedException
19
+ import io.imagestitcher.rn.ar.BackgroundRenderer
20
+ import io.imagestitcher.rn.ar.YuvImageConverter
21
+ import java.io.File
22
+ import java.util.concurrent.atomic.AtomicReference
23
+ import javax.microedition.khronos.egl.EGLConfig
24
+ import javax.microedition.khronos.opengles.GL10
25
+ import kotlin.math.atan
26
+ import kotlin.math.atan2
27
+ import kotlin.math.asin
28
+
29
+ /**
30
+ * Android twin of `RNSARCameraView.swift` (iOS Phase 4.4).
31
+ *
32
+ * Embeds a `GLSurfaceView` that renders the ARCore camera feed and
33
+ * drives the AR session's per-frame `update()` loop on the GL render
34
+ * thread. When the incremental stitcher is running, each frame's
35
+ * camera image is converted to JPEG and fed into the engine with the
36
+ * matching ARCore pose — full parity with iOS' ARSession path (no
37
+ * gyro fallback needed when this view is mounted).
38
+ *
39
+ * Why a GLSurfaceView and not a TextureView / SurfaceView:
40
+ * ARCore needs a GL_TEXTURE_EXTERNAL_OES texture as its camera
41
+ * sink (Session.setCameraTextureName). Only a GLSurfaceView with
42
+ * EGL14 context gives us the OES extension. The Renderer
43
+ * callback is also where the per-frame Session.update() lives, so
44
+ * the threading model lines up cleanly.
45
+ *
46
+ * Lifecycle:
47
+ * onAttachedToWindow → mark "wants to render", borrow Session
48
+ * from RNSARSession.instance
49
+ * onSurfaceCreated (GL) → create OES texture, build BackgroundRenderer
50
+ * onSurfaceChanged (GL) → notify session of display geometry
51
+ * onDrawFrame (GL) → session.update(); pose → log;
52
+ * if stitcher running: image → JPEG → engine
53
+ * onDetachedFromWindow → pause render thread; do NOT pause Session
54
+ * (other views may still be using it)
55
+ */
56
+ class RNSARCameraView @JvmOverloads constructor(
57
+ context: Context,
58
+ attrs: AttributeSet? = null,
59
+ defStyle: Int = 0,
60
+ ) : FrameLayout(context, attrs, defStyle), GLSurfaceView.Renderer {
61
+
62
+ private val glView: GLSurfaceView = GLSurfaceView(context).also { v ->
63
+ v.preserveEGLContextOnPause = true
64
+ v.setEGLContextClientVersion(2)
65
+ v.setEGLConfigChooser(8, 8, 8, 8, 16, 0)
66
+ v.setRenderer(this)
67
+ v.renderMode = GLSurfaceView.RENDERMODE_CONTINUOUSLY
68
+ }
69
+
70
+ private val backgroundRenderer = BackgroundRenderer()
71
+ private val sessionRef = AtomicReference<Session?>(null)
72
+ private var sessionTextureBound = false
73
+ /// Last known display rotation; consulted on each setDisplayGeometry
74
+ /// call so we can recompute when the user rotates the device.
75
+ private var lastDisplayRotation: Int = -1
76
+ private var surfaceWidth: Int = 0
77
+ private var surfaceHeight: Int = 0
78
+
79
+ /// Tmp directory for the per-frame JPEG file we hand to the
80
+ /// incremental engine. Created lazily and reused across frames
81
+ /// — no per-frame allocation.
82
+ private val tmpJpegFile: File by lazy {
83
+ File(context.cacheDir, "rlis-arframe.jpg")
84
+ }
85
+
86
+ /// Whether to feed the AR session's frames into the incremental
87
+ /// engine. Toggled by IncrementalStitcher.start/stop
88
+ /// via setIncrementalIngestionActive() below.
89
+ @Volatile private var ingestActive: Boolean = false
90
+
91
+ /// Pending takePhoto request, populated by `requestTakePhoto`
92
+ /// from the bridge thread and consumed by the GL render thread
93
+ /// on the next `onDrawFrame` so the latest ARCore frame is
94
+ /// captured. Cleared atomically so concurrent shutter taps
95
+ /// don't double-fire — the second tap's promise rejects the
96
+ /// older request before replacing it.
97
+ internal data class TakePhotoRequest(
98
+ val outputPath: String,
99
+ val quality: Int,
100
+ val promise: com.facebook.react.bridge.Promise,
101
+ )
102
+ private val pendingTakePhoto =
103
+ AtomicReference<TakePhotoRequest?>(null)
104
+
105
+ /// Called from the bridge (RNSARSession.takePhoto @ReactMethod).
106
+ /// Stores a request that will be fulfilled on the next render
107
+ /// tick. If another request is already queued, that one is
108
+ /// rejected (the JS layer should serialise its own calls).
109
+ internal fun requestTakePhoto(
110
+ outputPath: String,
111
+ quality: Int,
112
+ promise: com.facebook.react.bridge.Promise,
113
+ ) {
114
+ val req = TakePhotoRequest(outputPath, quality, promise)
115
+ val previous = pendingTakePhoto.getAndSet(req)
116
+ previous?.promise?.reject(
117
+ "ar-photo-superseded",
118
+ "takePhoto: superseded by a newer call before the frame was captured.",
119
+ )
120
+ // Wake the render loop in case it's idle.
121
+ glView.requestRender()
122
+ }
123
+
124
+ init {
125
+ addView(
126
+ glView,
127
+ LayoutParams(LayoutParams.MATCH_PARENT, LayoutParams.MATCH_PARENT),
128
+ )
129
+ }
130
+
131
+ override fun onAttachedToWindow() {
132
+ super.onAttachedToWindow()
133
+ Log.i(TAG, "onAttachedToWindow: requesting AR session start (iOS-parity didMoveToWindow)")
134
+ // iOS parity (didMoveToWindow): ensure the singleton AR
135
+ // session is running BEFORE we try to borrow it for
136
+ // rendering. Previously the view only borrowed an existing
137
+ // session — if nothing else had started one yet, the
138
+ // GLSurfaceView would stay at its cleared-black state
139
+ // forever and the user would see a black camera preview.
140
+ //
141
+ // startForView() is idempotent (no-op if a session is
142
+ // already running) and silently logs failures rather than
143
+ // throwing — if it returns false the view falls through to
144
+ // the borrow logic below, which then renders empty. Worst-
145
+ // case the user navigates away + back to retry.
146
+ RNSARSession.instance?.startForView()
147
+
148
+ glView.onResume()
149
+ // Try to borrow the session from the running RNSARSession.
150
+ val session = RNSARSession.instance?.getSessionForView()
151
+ if (session != null) {
152
+ sessionRef.set(session)
153
+ // ARCore's `Session.resume()` must be called on the main
154
+ // thread — startForView() above already resumed a freshly-
155
+ // created session, but if we got here with a pre-existing
156
+ // paused session (e.g. another ARCameraView's onDetached
157
+ // ran and paused, then this view re-mounted) we resume
158
+ // again here. Idempotent: Session.resume() is a no-op
159
+ // if the session is already resumed.
160
+ try {
161
+ session.resume()
162
+ } catch (e: CameraNotAvailableException) {
163
+ Log.w(TAG, "session.resume on attach: $e")
164
+ }
165
+ } else {
166
+ Log.w(
167
+ TAG,
168
+ "onAttachedToWindow: session is still null after startForView; " +
169
+ "preview will stay black until the view re-mounts " +
170
+ "(possible reasons: no Activity, ARCore install in progress, " +
171
+ "device unsupported — see RNSARSession logs)",
172
+ )
173
+ }
174
+ RNSARSession.instance?.bindCameraView(this)
175
+ IncrementalStitcher.bridgeInstance?.bindArCameraView(this)
176
+ }
177
+
178
+ override fun onDetachedFromWindow() {
179
+ super.onDetachedFromWindow()
180
+ Log.i(TAG, "onDetachedFromWindow: requesting AR session stop (iOS-parity didMoveToWindow)")
181
+ // Pause the GL thread so we stop drawing frames.
182
+ glView.onPause()
183
+ sessionTextureBound = false
184
+ IncrementalStitcher.bridgeInstance?.unbindArCameraView(this)
185
+ RNSARSession.instance?.unbindCameraView(this)
186
+ // iOS parity (didMoveToWindow else-branch): stop the session
187
+ // so the hardware camera is freed for vision-camera or other
188
+ // consumers when the user navigates away. Updated from the
189
+ // previous "do NOT pause the session" comment, which assumed
190
+ // the bridge module's start/stop owned lifecycle exclusively.
191
+ // With startForView()/stopForView(), the view is now the
192
+ // primary lifecycle owner for the auto-mounted case (the
193
+ // most common path: AuditCaptureScreen mounts the view, which
194
+ // starts the session; navigates away, which stops it).
195
+ // The JS-facing `start(promise)` / `stop(promise)` continue
196
+ // to work for hosts that prefer explicit control — the
197
+ // refs/state are shared.
198
+ RNSARSession.instance?.stopForView()
199
+ }
200
+
201
+ /// Called by IncrementalStitcher.start/stop. When true,
202
+ /// each ARCore frame's camera image is encoded to JPEG + handed
203
+ /// to the engine; when false, the per-frame work skips ingestion
204
+ /// (the camera feed continues to render either way).
205
+ fun setIncrementalIngestionActive(active: Boolean) {
206
+ // P3-G diagnostic — surfaces whether the camera view ever
207
+ // got "engage the ingestion path" command from the stitcher.
208
+ // Common failure mode: stitcher.start() ran, but
209
+ // arCameraViewRef was null (view not bound yet) → this never
210
+ // fires → forwardToIncremental never runs → 0 frames.
211
+ Log.i(TAG, "setIncrementalIngestionActive: $active (prev=$ingestActive)")
212
+ ingestActive = active
213
+ }
214
+
215
+ // ── GLSurfaceView.Renderer ─────────────────────────────────────
216
+
217
+ override fun onSurfaceCreated(gl: GL10?, config: EGLConfig?) {
218
+ GLES20.glClearColor(0f, 0f, 0f, 1f)
219
+ backgroundRenderer.createOnGlThread()
220
+ sessionTextureBound = false
221
+ }
222
+
223
+ override fun onSurfaceChanged(gl: GL10?, width: Int, height: Int) {
224
+ GLES20.glViewport(0, 0, width, height)
225
+ surfaceWidth = width
226
+ surfaceHeight = height
227
+ applyDisplayGeometry()
228
+ }
229
+
230
+ override fun onDrawFrame(gl: GL10?) {
231
+ GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT or GLES20.GL_DEPTH_BUFFER_BIT)
232
+
233
+ val session = sessionRef.get() ?: run {
234
+ // Session not yet attached (start() hasn't run, or
235
+ // the bridge module instance was rebuilt). Try once
236
+ // more in case the bridge resolved it after onAttach.
237
+ val late = RNSARSession.instance?.getSessionForView()
238
+ if (late != null) sessionRef.set(late)
239
+ return
240
+ }
241
+ if (!sessionTextureBound) {
242
+ backgroundRenderer.bindToSession(session)
243
+ sessionTextureBound = true
244
+ // Ensure ARCore knows the surface geometry.
245
+ applyDisplayGeometry()
246
+ }
247
+
248
+ val frame = try {
249
+ session.update()
250
+ } catch (e: SessionPausedException) {
251
+ return // session paused — wait for resume
252
+ } catch (t: Throwable) {
253
+ Log.w(TAG, "session.update failed: ${t.message}")
254
+ return
255
+ }
256
+
257
+ // Draw the camera background regardless of tracking state —
258
+ // gives the user something to look at while AR initialises.
259
+ backgroundRenderer.draw(frame)
260
+
261
+ val camera: Camera = frame.camera
262
+
263
+ // ── V15.0e — vertical plane detection (iOS parity) ──────────
264
+ // Run this each frame so the JS 2 Hz getARPlaneStatus poll
265
+ // sees a live answer without the user having to take any
266
+ // action. iOS ARKit re-runs evaluation internally on each
267
+ // ARSessionDelegate didUpdate callback; we mirror by polling
268
+ // every ARCore frame. Cost: ~10-20 us per frame at idle (no
269
+ // planes), ~50-100 us when iterating a handful of tracking
270
+ // planes — negligible against the 16ms frame budget.
271
+ val pose = camera.pose
272
+ // ARCore Pose convention: zAxis is the world-space direction
273
+ // of the local Z axis. Camera looks down -Z (OpenGL
274
+ // convention), so cameraForward = -zAxis. ARCore 1.45's
275
+ // Pose.getZAxis() takes no args and returns a new FloatArray.
276
+ val zAxis = pose.zAxis
277
+ val cameraForwardWorld = floatArrayOf(-zAxis[0], -zAxis[1], -zAxis[2])
278
+ val cameraPosWorld = floatArrayOf(pose.tx(), pose.ty(), pose.tz())
279
+ RNSARSession.instance?.evaluatePlanesForFrame(
280
+ cameraForwardWorld,
281
+ cameraPosWorld,
282
+ )
283
+
284
+ // Push pose into the AR session log. Mirrors iOS' delegate
285
+ // path; the existing RNSARFramePose / appendPose
286
+ // contract was already in place for Phase 4.
287
+ appendPose(camera, frame.timestamp)
288
+
289
+ // Forward to the incremental stitcher if engaged.
290
+ if (ingestActive) {
291
+ forwardToIncremental(frame, camera)
292
+ }
293
+
294
+ // takePhoto consumer — runs on EVERY render tick (not just
295
+ // when ingest is active), since the host calls takePhoto in
296
+ // photo mode where ingest is off. No-op when no request is
297
+ // pending; cheap atomic CAS on the hot path.
298
+ pendingTakePhoto.getAndSet(null)?.let { req ->
299
+ fulfilTakePhoto(frame, req)
300
+ }
301
+ }
302
+
303
+ /// Capture the current ARCore frame to JPEG and resolve / reject
304
+ /// `req.promise`. Runs on the GL render thread, called from
305
+ /// `onDrawFrame` after the frame has been obtained via
306
+ /// `session.update()`. Mirrors iOS' `RNSARSession.takePhoto`
307
+ /// resolution shape: `{ path, width, height, isMirrored,
308
+ /// isRawPhoto }` so JS code is platform-agnostic.
309
+ private fun fulfilTakePhoto(
310
+ frame: com.google.ar.core.Frame,
311
+ req: TakePhotoRequest,
312
+ ) {
313
+ val image = try {
314
+ frame.acquireCameraImage()
315
+ } catch (t: Throwable) {
316
+ req.promise.reject(
317
+ "ar-photo-no-frame",
318
+ "takePhoto: acquireCameraImage failed: ${t.message}",
319
+ )
320
+ return
321
+ }
322
+ val width = image.width
323
+ val height = image.height
324
+ try {
325
+ val written = YuvImageConverter.encodeToJpeg(
326
+ image,
327
+ req.outputPath,
328
+ jpegQuality = req.quality.coerceIn(1, 100),
329
+ displayRotation = if (lastDisplayRotation >= 0)
330
+ lastDisplayRotation
331
+ else
332
+ Surface.ROTATION_0,
333
+ )
334
+ if (written == null) {
335
+ req.promise.reject(
336
+ "ar-photo-encode-failed",
337
+ "takePhoto: YuvImageConverter.encodeToJpeg returned null.",
338
+ )
339
+ return
340
+ }
341
+ val result = com.facebook.react.bridge.Arguments.createMap().apply {
342
+ putString("path", written)
343
+ putInt("width", width)
344
+ putInt("height", height)
345
+ putBoolean("isMirrored", false)
346
+ putBoolean("isRawPhoto", false)
347
+ }
348
+ req.promise.resolve(result)
349
+ } catch (t: Throwable) {
350
+ req.promise.reject(
351
+ "ar-photo-failed",
352
+ "takePhoto: unexpected error: ${t.message}",
353
+ t,
354
+ )
355
+ } finally {
356
+ // Image must always be closed or ARCore will starve.
357
+ try { image.close() } catch (_: Throwable) {}
358
+ }
359
+ }
360
+
361
+ private fun appendPose(camera: Camera, timestampNs: Long) {
362
+ val pose = camera.pose
363
+ val translation = pose.translation
364
+ val rotation = pose.rotationQuaternion // x, y, z, w
365
+ val intrinsics = camera.imageIntrinsics
366
+ val focal = intrinsics.focalLength
367
+ val principal = intrinsics.principalPoint
368
+ val dims = intrinsics.imageDimensions
369
+
370
+ val tracking = when (camera.trackingState) {
371
+ TrackingState.TRACKING -> RNSARSession.TRACKING_TRACKING
372
+ TrackingState.PAUSED -> RNSARSession.TRACKING_LIMITED
373
+ TrackingState.STOPPED -> RNSARSession.TRACKING_NOT_AVAILABLE
374
+ else -> RNSARSession.TRACKING_NOT_AVAILABLE
375
+ }
376
+
377
+ val framePose = RNSARFramePose(
378
+ tx = translation[0].toDouble(),
379
+ ty = translation[1].toDouble(),
380
+ tz = translation[2].toDouble(),
381
+ qx = rotation[0].toDouble(),
382
+ qy = rotation[1].toDouble(),
383
+ qz = rotation[2].toDouble(),
384
+ qw = rotation[3].toDouble(),
385
+ fx = focal[0].toDouble(),
386
+ fy = focal[1].toDouble(),
387
+ cx = principal[0].toDouble(),
388
+ cy = principal[1].toDouble(),
389
+ imageWidth = dims[0],
390
+ imageHeight = dims[1],
391
+ timestampMs = timestampNs / 1_000_000.0,
392
+ trackingState = tracking,
393
+ )
394
+ RNSARSession.instance?.appendPose(framePose)
395
+ RNSARSession.instance?.updateTrackingState(camera.trackingState)
396
+ }
397
+
398
+ /// P3-G diagnostic — rate-limit the per-frame log so we can see
399
+ /// at-a-glance whether forwardToIncremental is even running, vs
400
+ /// being short-circuited at the `if (ingestActive)` guard in
401
+ /// onDrawFrame.
402
+ private var forwardLogTick: Int = 0
403
+
404
+ private fun forwardToIncremental(
405
+ frame: com.google.ar.core.Frame,
406
+ camera: Camera,
407
+ ) {
408
+ if (forwardLogTick++ % 30 == 0) {
409
+ Log.i(TAG, "forwardToIncremental: ingestActive=$ingestActive trackingState=${camera.trackingState}")
410
+ }
411
+ // Acquire the camera image. Each call may throw
412
+ // NotYetAvailableException for the first ~1-2 frames before
413
+ // ARCore catches up — silently skip those.
414
+ val image = try {
415
+ frame.acquireCameraImage()
416
+ } catch (t: Throwable) {
417
+ if (forwardLogTick % 30 == 1) {
418
+ Log.w(TAG, "forwardToIncremental: acquireCameraImage failed: ${t.message}")
419
+ }
420
+ return
421
+ }
422
+ try {
423
+ val written = YuvImageConverter.encodeToJpeg(
424
+ image,
425
+ tmpJpegFile.absolutePath,
426
+ jpegQuality = 70,
427
+ // 2026-05-15 (B3) — pass current display rotation so
428
+ // the encoded JPEG gets an EXIF orientation tag.
429
+ // Without this, the live thumbnail strip shows
430
+ // sideways pictures when the device is held in
431
+ // portrait (sensor pixels are landscape by default).
432
+ // lastDisplayRotation is updated by the
433
+ // updateDisplayRotation() helper called from
434
+ // didMoveToWindow / the ARCore Session.setDisplayGeometry
435
+ // hook (see line ~410).
436
+ displayRotation = if (lastDisplayRotation >= 0)
437
+ lastDisplayRotation else android.view.Surface.ROTATION_0,
438
+ ) ?: return
439
+
440
+ // Compute yaw + pitch from the ARCore quaternion using
441
+ // the same convention the iOS Swift side uses (camera-
442
+ // forward in world space). This keeps the two platforms
443
+ // numerically aligned for the FoV-overlap gate.
444
+ val q = camera.pose.rotationQuaternion // x, y, z, w
445
+ val (yaw, pitch) = quaternionYawPitch(q)
446
+
447
+ // Both FoVs + the full quaternion + intrinsics go to the
448
+ // engine. V6 pose-driven path uses (qx, qy, qz, qw, fx,
449
+ // fy, cx, cy, w, h) to compute the geometrically-exact
450
+ // homography.
451
+ val intrinsics = camera.imageIntrinsics
452
+ val fx = intrinsics.focalLength[0].toDouble()
453
+ val fy = intrinsics.focalLength[1].toDouble()
454
+ val cxIntr = intrinsics.principalPoint[0].toDouble()
455
+ val cyIntr = intrinsics.principalPoint[1].toDouble()
456
+ val w = intrinsics.imageDimensions[0].toDouble()
457
+ val h = intrinsics.imageDimensions[1].toDouble()
458
+ val fovHRad = 2.0 * atan(w / (2.0 * fx))
459
+ val fovVRad = 2.0 * atan(h / (2.0 * fy))
460
+ val fovHDeg = fovHRad * 180.0 / Math.PI
461
+ val fovVDeg = fovVRad * 180.0 / Math.PI
462
+
463
+ // ARCore quaternion comes back in (x, y, z, w) order.
464
+ val qarr = camera.pose.rotationQuaternion
465
+ // P3-F: also extract translation so the KeyframeGate's
466
+ // plane-based ray-projection can compute polygon overlap.
467
+ // Previously these were dropped, forcing the gate into
468
+ // angular-fallback even when a plane was latched.
469
+ val tArr = camera.pose.translation
470
+
471
+ val trackingPoor = camera.trackingState != TrackingState.TRACKING
472
+ postFrameToEngine(
473
+ path = written,
474
+ tx = tArr[0].toDouble(),
475
+ ty = tArr[1].toDouble(),
476
+ tz = tArr[2].toDouble(),
477
+ qx = qarr[0].toDouble(), qy = qarr[1].toDouble(),
478
+ qz = qarr[2].toDouble(), qw = qarr[3].toDouble(),
479
+ fx = fx, fy = fy, cx = cxIntr, cy = cyIntr,
480
+ imageWidth = intrinsics.imageDimensions[0],
481
+ imageHeight = intrinsics.imageDimensions[1],
482
+ yaw = yaw, pitch = pitch,
483
+ fovHorizDegrees = fovHDeg, fovVertDegrees = fovVDeg,
484
+ trackingPoor = trackingPoor,
485
+ )
486
+ } finally {
487
+ image.close()
488
+ }
489
+ }
490
+
491
+ private fun postFrameToEngine(
492
+ path: String,
493
+ tx: Double, ty: Double, tz: Double,
494
+ qx: Double, qy: Double, qz: Double, qw: Double,
495
+ fx: Double, fy: Double, cx: Double, cy: Double,
496
+ imageWidth: Int, imageHeight: Int,
497
+ yaw: Double,
498
+ pitch: Double,
499
+ fovHorizDegrees: Double,
500
+ fovVertDegrees: Double,
501
+ trackingPoor: Boolean,
502
+ ) {
503
+ val module = IncrementalStitcher.bridgeInstance ?: return
504
+ module.ingestFromARCameraView(
505
+ path = path,
506
+ tx = tx, ty = ty, tz = tz,
507
+ qx = qx, qy = qy, qz = qz, qw = qw,
508
+ fx = fx, fy = fy, cx = cx, cy = cy,
509
+ imageWidth = imageWidth, imageHeight = imageHeight,
510
+ yaw = yaw,
511
+ pitch = pitch,
512
+ fovHorizDegrees = fovHorizDegrees,
513
+ fovVertDegrees = fovVertDegrees,
514
+ trackingPoor = trackingPoor,
515
+ )
516
+ }
517
+
518
+ private fun applyDisplayGeometry() {
519
+ val session = sessionRef.get() ?: return
520
+ val rotation = (context.getSystemService(Context.WINDOW_SERVICE) as? WindowManager)
521
+ ?.defaultDisplay
522
+ ?.rotation
523
+ ?: Surface.ROTATION_0
524
+ if (rotation != lastDisplayRotation
525
+ || surfaceWidth > 0 || surfaceHeight > 0
526
+ ) {
527
+ session.setDisplayGeometry(rotation, surfaceWidth, surfaceHeight)
528
+ lastDisplayRotation = rotation
529
+ }
530
+ }
531
+
532
+ /**
533
+ * Convert an ARCore quaternion (x, y, z, w) to (yaw, pitch) in
534
+ * radians — same convention as the iOS Swift side: rotate the
535
+ * camera-forward (-Z) vector by the quaternion, then pull yaw
536
+ * (atan2 onto X-Z plane) and pitch (asin of Y component).
537
+ *
538
+ * Closed-form forward = R * (0, 0, -1), where R is the standard
539
+ * quaternion-to-3x3 matrix. Bottom row of R gives `forwardZ`,
540
+ * etc; multiplying (0,0,-1) just negates the third column.
541
+ */
542
+ private fun quaternionYawPitch(q: FloatArray): Pair<Double, Double> {
543
+ val x = q[0].toDouble()
544
+ val y = q[1].toDouble()
545
+ val z = q[2].toDouble()
546
+ val w = q[3].toDouble()
547
+ val forwardX = -(2.0 * (x * z + w * y))
548
+ val forwardY = -(2.0 * (y * z - w * x))
549
+ val forwardZ = -(1.0 - 2.0 * (x * x + y * y))
550
+ val yaw = atan2(forwardX, -forwardZ)
551
+ val pitch = asin(forwardY.coerceIn(-1.0, 1.0))
552
+ return yaw to pitch
553
+ }
554
+
555
+ companion object {
556
+ private const val TAG = "RNSARCameraView"
557
+ }
558
+ }
@@ -0,0 +1,35 @@
1
+ // SPDX-License-Identifier: Apache-2.0
2
+ package io.imagestitcher.rn
3
+
4
+ import com.facebook.react.uimanager.SimpleViewManager
5
+ import com.facebook.react.uimanager.ThemedReactContext
6
+
7
+ /**
8
+ * RN ViewManager for `RNSARCameraView`.
9
+ *
10
+ * JS side imports it as:
11
+ *
12
+ * import { requireNativeComponent } from 'react-native';
13
+ * const RNARCamera = requireNativeComponent('RNSARCameraView');
14
+ *
15
+ * Or — preferred — uses the SDK's existing `<ARCameraView>` wrapper
16
+ * in `src/camera/ARCameraView.tsx` which auto-selects the native
17
+ * component for iOS vs Android.
18
+ *
19
+ * The view itself is config-free for now (no JS-side props beyond
20
+ * `style`) since lifecycle is driven by mount/unmount + the
21
+ * incremental stitcher's start/finalize methods. Future phases may
22
+ * add props like `enabled` to allow JS-controlled pause/resume of
23
+ * the GL render loop.
24
+ */
25
+ class RNSARCameraViewManager : SimpleViewManager<RNSARCameraView>() {
26
+
27
+ override fun getName(): String = REACT_CLASS
28
+
29
+ override fun createViewInstance(reactContext: ThemedReactContext): RNSARCameraView =
30
+ RNSARCameraView(reactContext)
31
+
32
+ companion object {
33
+ const val REACT_CLASS = "RNSARCameraView"
34
+ }
35
+ }