react-native-image-stitcher 0.14.2 → 0.15.1

package/android/src/main/java/io/imagestitcher/rn/KeyframeGate.kt

@@ -140,6 +140,23 @@ internal class KeyframeGate : AutoCloseable {
             nativeSetFlowMaxTranslationM(nativeHandle, value)
         }
 
+    /// Wall-clock keyframe-interval budget, in MILLISECONDS, between
+    /// consecutive accepted keyframes before force-acceptance.  Same
+    /// knob iOS exposes via setMaxKeyframeIntervalMs (KeyframeGate.swift
+    /// `maxKeyframeIntervalMs`).  Unlike flowMaxTranslationM this applies
+    /// to BOTH the Pose and Flow strategies, and is passed STRAIGHT
+    /// THROUGH (already in the unit the C++ expects — no conversion).
+    /// Default 2000 ms (matches iOS); 0 = disabled.  The C++ setter
+    /// clamps to ≥ 0.  NOTE: like every other facade property the
+    /// initializer below does NOT fire this setter, so the caller
+    /// (IncrementalStitcher.kt) writes it explicitly at capture start
+    /// to push the value into C++ (same contract as the iOS facade).
+    var maxKeyframeIntervalMs: Double = 2000.0
+        set(value) {
+            field = value
+            nativeSetMaxKeyframeIntervalMs(nativeHandle, value)
+        }
+
     /// 2026-05-22 (audit F5) — Flow strategy: Shi-Tomasi max corners
     /// to track per frame.  Same knob iOS exposes via setFlowMaxCorners.
     /// C++ clamps to ≥ 30.  Higher = more sensitive to fine detail but
@@ -305,6 +322,9 @@ internal class KeyframeGate : AutoCloseable {
     private external fun nativeSetDisableAngularFallback(handle: Long, disabled: Boolean)
     private external fun nativeSetFlowNoveltyPercentile(handle: Long, percentile: Double)
     private external fun nativeSetFlowMaxTranslationM(handle: Long, metres: Double)
+    // Wall-clock keyframe-interval budget (ms).  iOS parity:
+    // KeyframeGateBridge.setMaxKeyframeIntervalMs.
+    private external fun nativeSetMaxKeyframeIntervalMs(handle: Long, ms: Double)
     // 2026-05-22 (audit F5) — flow-strategy tunables that were
     // previously iOS-only.  Add Android JNI parity so the Settings UI
     // sliders work on both platforms.
@@ -362,6 +382,15 @@ internal class KeyframeGate : AutoCloseable {
             9 -> "max-reached"
             10 -> "overlap-too-high"
             11 -> "overlap-too-high (angular)"
+            // Flow-strategy reasons (v0.3.0, cpp KeyframeGateDecisionReason
+            // 12-15) — strings must match the cpp/iOS labels exactly.
+            12 -> "ok-flow"
+            13 -> "first-flow"
+            14 -> "overlap-too-high (flow)"
+            15 -> "ok-flow-translation"
+            // Wall-clock keyframe-interval force-accept (Pose + Flow);
+            // cpp KeyframeGateDecisionReason::AcceptTimeInterval = 16.
+            16 -> "ok-time-interval"
             else -> "unknown($code)"
         }
     }

package/android/src/main/java/io/imagestitcher/rn/RNImageStitcherPackage.kt

@@ -101,10 +101,6 @@ class RNImageStitcherPackage : ReactPackage {
             RNSARSession(reactContext),
             IncrementalStitcher(reactContext),
             FileBridge(reactContext),
-            // v0.8.0 Phase 4b.ii — Android JSI installer for the
-            // host-worklet `__stitcherProxy` global.  Mirror of
-            // iOS' `StitcherJsiInstaller`.
-            StitcherJsiInstallerModule(reactContext),
         )
     }
 

package/android/src/main/java/io/imagestitcher/rn/RNSARCameraView.kt

@@ -58,6 +58,13 @@ class RNSARCameraView @JvmOverloads constructor(
     defStyle: Int = 0,
 ) : FrameLayout(context, attrs, defStyle), GLSurfaceView.Renderer {
 
+    // Raw camera sensor aspect ratio (W÷H, always > 1 for landscape sensors).
+    // Initialised to 4:3 — a safe fallback for the first layout pass before
+    // the session is attached.  Updated from session.cameraConfig once the
+    // session is available; many Android ARCore devices use 16:9 configs
+    // (e.g. Pixel phones), so reading it dynamically is important here.
+    private var cameraAspect: Float = 4f / 3f
+
     private val glView: GLSurfaceView = GLSurfaceView(context).also { v ->
         v.preserveEGLContextOnPause = true
         v.setEGLContextClientVersion(2)
@@ -123,6 +130,10 @@ class RNSARCameraView @JvmOverloads constructor(
     }
 
     init {
+        // Black background avoids a flash before the GL surface starts
+        // clearing itself black each frame (the GL-level letterbox draws
+        // the bars; this is just belt-and-suspenders for the first frame).
+        setBackgroundColor(android.graphics.Color.BLACK)
         addView(
             glView,
             LayoutParams(LayoutParams.MATCH_PARENT, LayoutParams.MATCH_PARENT),
@@ -163,6 +174,26 @@ class RNSARCameraView @JvmOverloads constructor(
             } catch (e: CameraNotAvailableException) {
                 Log.w(TAG, "session.resume on attach: $e")
             }
+            // Read the actual camera image dimensions from the ARCore
+            // session config so the GL-level letterbox can size its box.
+            // cameraConfig is stable after session creation; on Pixel and
+            // some other Android devices the default config is 16:9, not
+            // 4:3, so we must read dynamically rather than hard-code.
+            try {
+                val size = session.cameraConfig.imageSize
+                if (size.width > 0 && size.height > 0) {
+                    cameraAspect = size.width.toFloat() / size.height.toFloat()
+                    Log.i(TAG, "cameraConfig imageSize: ${size.width}×${size.height} → cameraAspect=$cameraAspect")
+                    // Invalidate the cached display geometry so the next
+                    // onDrawFrame re-pushes it with the now-known camera
+                    // aspect.  The GL-level letterbox recomputes the box
+                    // every frame — no view resize needed.
+                    lastGeomW = -1
+                    lastGeomH = -1
+                }
+            } catch (t: Throwable) {
+                Log.w(TAG, "cameraConfig not yet available in onAttach; will use $cameraAspect fallback: ${t.message}")
+            }
         } else {
             Log.w(
                 TAG,
@@ -222,6 +253,54 @@ class RNSARCameraView @JvmOverloads constructor(
         ingestActive = active
     }
 
+    // ── GL-level letterbox ─────────────────────────────────────────
+    //
+    // The [glView] stays full-screen (MATCH_PARENT); we letterbox at the
+    // GL layer instead of resizing the SurfaceView.  Resizing the view
+    // does NOT work for ARCore: its BackgroundRenderer maps the camera
+    // texture with `Frame.transformCoordinates2d`, which uses the
+    // session's *display geometry* — not the view bounds.  A resized view
+    // therefore still rendered the full-screen (centre-cropped) camera,
+    // merely clipped to the smaller view → a cropped scene with one
+    // visible bar (the other hidden behind the capture controls).
+    //
+    // The correct fix is pure GL + ARCore geometry, applied per frame:
+    //   1. clear the WHOLE surface to black  → the letterbox bars,
+    //   2. setDisplayGeometry to the BOX size → ARCore's UV transform
+    //      fills the box aspect; when box aspect == camera aspect there
+    //      is nothing to crop, so the full FOV shows,
+    //   3. glViewport to the centred box      → camera draws only there.
+
+    /** Last display geometry pushed to ARCore; only re-push on change. */
+    private var lastGeomW: Int = -1
+    private var lastGeomH: Int = -1
+    private var lastGeomRotation: Int = -1
+
+    /**
+     * The centred letterbox rect [x, y, w, h] inside the full GL surface
+     * that preserves the camera's content aspect ratio.  The sensor is
+     * landscape (e.g. 640×480, 4:3); in portrait the on-screen content
+     * aspect is the inverse, so [cameraAspect] is inverted when the
+     * surface is taller than wide.  Falls back to the full surface until
+     * the surface has been measured.
+     */
+    private fun letterboxBox(): IntArray {
+        val sw = surfaceWidth
+        val sh = surfaceHeight
+        if (sw <= 0 || sh <= 0 || cameraAspect <= 0f) return intArrayOf(0, 0, sw, sh)
+        val contentAspect = if (sh > sw) 1f / cameraAspect else cameraAspect
+        val surfaceAspect = sw.toFloat() / sh.toFloat()
+        return if (surfaceAspect > contentAspect) {
+            // Surface wider than content — vertical bars left/right.
+            val w = (sh * contentAspect).toInt()
+            intArrayOf((sw - w) / 2, 0, w, sh)
+        } else {
+            // Surface taller than content — horizontal bars top/bottom.
+            val h = (sw / contentAspect).toInt()
+            intArrayOf(0, (sh - h) / 2, sw, h)
+        }
+    }
+
     // ── GLSurfaceView.Renderer ─────────────────────────────────────
 
     override fun onSurfaceCreated(gl: GL10?, config: EGLConfig?) {
@@ -238,6 +317,10 @@ class RNSARCameraView @JvmOverloads constructor(
     }
 
     override fun onDrawFrame(gl: GL10?) {
+        // Step 1 — paint the WHOLE surface black.  This is the letterbox:
+        // anything outside the camera box below stays black.
+        GLES20.glViewport(0, 0, surfaceWidth, surfaceHeight)
+        GLES20.glClearColor(0f, 0f, 0f, 1f)
         GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT or GLES20.GL_DEPTH_BUFFER_BIT)
 
         val session = sessionRef.get() ?: run {
@@ -251,10 +334,14 @@ class RNSARCameraView @JvmOverloads constructor(
         if (!sessionTextureBound) {
             backgroundRenderer.bindToSession(session)
             sessionTextureBound = true
-            // Ensure ARCore knows the surface geometry.
-            applyDisplayGeometry()
         }
 
+        // Step 2 — keep ARCore's display geometry equal to the letterbox
+        // box (not the full surface) so its UV transform fills the box
+        // aspect with the full camera FOV (no centre-crop).  Cheap: only
+        // calls setDisplayGeometry when the box actually changes.
+        applyDisplayGeometry()
+
         val frame = try {
             session.update()
         } catch (e: SessionPausedException) {
@@ -264,6 +351,11 @@ class RNSARCameraView @JvmOverloads constructor(
             return
         }
 
+        // Step 3 — confine the camera draw to the centred box; the black
+        // cleared in step 1 remains as the bars around it.
+        val box = letterboxBox()
+        GLES20.glViewport(box[0], box[1], box[2], box[3])
+
         // Draw the camera background regardless of tracking state —
         // gives the user something to look at while AR initialises.
         backgroundRenderer.draw(frame)
@@ -296,20 +388,12 @@ class RNSARCameraView @JvmOverloads constructor(
         // contract was already in place for Phase 4.
         appendPose(camera, frame.timestamp)
 
-        // Forward to the incremental stitcher if engaged, OR if any
-        // host worklets are registered (v0.8.0 Phase 4b.iii).  iOS'
-        // `RNSARWorkletRuntime.dispatchFrame:pose:` fires on every
-        // AR frame regardless of capture state; Android needs the
-        // same semantic so host worklets see the AR-mode preview
-        // stream, not just capture frames.
-        //
-        // `hasHostWorklets()` is a microsecond atomic-read on the
-        // native registry — cheap enough to hit per frame.  When
-        // no host worklets are registered AND no capture is active,
-        // the entire forwardToIncremental branch (including the
-        // ~3-5ms NV21 pack) is skipped — same cost envelope as
-        // before Phase 4b.iii.
-        if (ingestActive || StitcherWorkletRuntime.hasHostWorklets()) {
+        // Forward to the incremental stitcher only when capture is
+        // engaged.  (The v0.8.0 host-worklet dispatch — which also
+        // forwarded preview frames whenever host worklets were
+        // registered — was archived in the 2026-06 batch-keyframe
+        // cleanup.)
+        if (ingestActive) {
             forwardToIncremental(frame, camera)
         }
 
@@ -541,22 +625,14 @@ class RNSARCameraView @JvmOverloads constructor(
         //  written to `tmpJpegFile`, passed as `legacyJpegPath`.
         //  See the v0.3 / F8.6 entries in CHANGELOG.md.)
         //
-        // v0.8.0 Phase 3c — route through the worklet runtime's
-        // `runFirstParty` indirection.  The lambda body is the
-        // unchanged engine ingest call; the indirection sets up
-        // the seam where Phase 4 will fan out to host worklets
-        // without touching this first-party path.  Synchronous
-        // invocation preserves the ARCore Image ownership contract
-        // — the engine consumes the TransferredNV21 inside the
-        // lambda before ARCore recycles the Image.
-        StitcherWorkletRuntime.installIfNeeded()
-        // v0.8.0 Phase 4b.iii — only run first-party stitching when
-        // the host has actively engaged capture (`setIncrementalIngestionActive(true)`).
-        // The host-worklet dispatch below runs regardless, so AR-mode
-        // preview frames stream through registered host worklets even
-        // before/after capture.
+        // Synchronous engine ingest.  The ARCore Image ownership
+        // contract requires the engine to consume the TransferredNV21
+        // before ARCore recycles the Image, so this runs inline.  Only
+        // ingest when the host has actively engaged capture
+        // (`setIncrementalIngestionActive(true)`).  (The v0.8.0 worklet-
+        // runtime `runFirstParty` indirection + host-worklet fan-out
+        // were archived in the 2026-06 batch-keyframe cleanup.)
         if (ingestActive) {
-        StitcherWorkletRuntime.runFirstParty {
         module.ingestFromARCameraView(
             tx = tArr[0].toDouble(),
             ty = tArr[1].toDouble(),
@@ -608,42 +684,7 @@ class RNSARCameraView @JvmOverloads constructor(
                 ) != null
             },
         )
-        }  // closes StitcherWorkletRuntime.runFirstParty { … } (v0.8.0 Phase 3c)
         }  // closes `if (ingestActive)` (v0.8.0 Phase 4b.iii)
-
-        // ── v0.8.0 Phase 4b.iii — host-worklet fan-out ─────────────
-        //
-        // Dispatch the AR frame to every host worklet registered via
-        // `globalThis.__stitcherProxy.install(workletFn)` (the
-        // `useFrameProcessor` hook's AR-mode path).  The native side
-        // fast-path early-exits when the registry is empty (~ns
-        // cost), so this call is free for first-party-only deployments.
-        //
-        // Map the trackingState back to the JS-visible string set.
-        // `RNSARSession.TRACKING_*` are int codes; we re-derive the
-        // string here instead of plumbing it through.  (Could be
-        // refactored into a helper if/when other call sites need
-        // it.)
-        val trackingStateStr = when (camera.trackingState) {
-            TrackingState.TRACKING -> "normal"
-            TrackingState.PAUSED -> "limited"
-            TrackingState.STOPPED -> "notAvailable"
-            else -> ""
-        }
-        StitcherWorkletRuntime.dispatchToHostWorklets(
-            nv21Bytes = packed.nv21,
-            width = packed.width,
-            height = packed.height,
-            qx = qarr[0].toDouble(),
-            qy = qarr[1].toDouble(),
-            qz = qarr[2].toDouble(),
-            qw = qarr[3].toDouble(),
-            tx = tArr[0].toDouble(),
-            ty = tArr[1].toDouble(),
-            tz = tArr[2].toDouble(),
-            timestampNs = frame.timestamp.toDouble(),
-            trackingState = trackingStateStr,
-        )
     }
 
     /// v0.13.2 — map the JS physical device orientation to the
@@ -666,11 +707,28 @@ class RNSARCameraView @JvmOverloads constructor(
             ?.defaultDisplay
             ?.rotation
             ?: Surface.ROTATION_0
-        if (rotation != lastDisplayRotation
-            || surfaceWidth > 0 || surfaceHeight > 0
-        ) {
-            session.setDisplayGeometry(rotation, surfaceWidth, surfaceHeight)
-            lastDisplayRotation = rotation
+        // Keep lastDisplayRotation current regardless — the JPEG encode
+        // path (forwardToIncremental → encodeJpegFromNV21) reads it for
+        // the EXIF orientation tag.
+        lastDisplayRotation = rotation
+
+        val box = letterboxBox()
+        val bw = box[2]
+        val bh = box[3]
+        if (bw <= 0 || bh <= 0) return
+        // Feed ARCore the BOX dimensions (not the full surface) so its UV
+        // transform fills the box aspect — the full camera FOV with no
+        // centre-crop.  Only push on change to avoid per-frame churn.
+        if (rotation != lastGeomRotation || bw != lastGeomW || bh != lastGeomH) {
+            session.setDisplayGeometry(rotation, bw, bh)
+            lastGeomRotation = rotation
+            lastGeomW = bw
+            lastGeomH = bh
+            Log.d(
+                TAG,
+                "setDisplayGeometry(box): rotation=$rotation box=${bw}×${bh} "
+                    + "surface=${surfaceWidth}×${surfaceHeight} cameraAspect=$cameraAspect",
+            )
         }
     }
 

package/android/src/main/java/io/imagestitcher/rn/RNSARSession.kt

@@ -11,6 +11,8 @@ import com.facebook.react.bridge.ReactApplicationContext
 import com.facebook.react.bridge.ReactContextBaseJavaModule
 import com.facebook.react.bridge.ReactMethod
 import com.google.ar.core.ArCoreApk
+import com.google.ar.core.CameraConfig
+import com.google.ar.core.CameraConfigFilter
 import com.google.ar.core.Config
 import com.google.ar.core.Plane
 import com.google.ar.core.Pose
@@ -164,6 +166,7 @@ class RNSARSession(reactContext: ReactApplicationContext)
 
             val session = sessionRef.get() ?: Session(reactApplicationContext).also {
                 sessionRef.set(it)
+                selectMatchingCameraConfig(it)
             }
             val config = Config(session).apply {
                 // Smoothed depth is the ARCore equivalent of iOS
@@ -322,6 +325,7 @@ class RNSARSession(reactContext: ReactApplicationContext)
 
             val session = Session(reactApplicationContext).also {
                 sessionRef.set(it)
+                selectMatchingCameraConfig(it)
             }
             val config = Config(session).apply {
                 if (session.isDepthModeSupported(Config.DepthMode.AUTOMATIC)) {
@@ -832,6 +836,51 @@ class RNSARSession(reactContext: ReactApplicationContext)
         poseLogLock.write { poseLog.clear() }
     }
 
+    /**
+     * Pick an ARCore camera config whose CPU image and GPU texture share
+     * the same aspect ratio, so the preview (texture) and the captured /
+     * stitched frames (acquireCameraImage) cover the SAME field of view.
+     *
+     * ARCore's default often pairs a 16:9 GPU texture with a 4:3 CPU
+     * image (e.g. 1920x1080 texture + 640x480 image on the Galaxy A35):
+     * the texture is then missing ~12 deg of vertical sensor FOV the
+     * image has, so the preview can never match the photo.  Choosing a
+     * config where the two aspects match (preferring 4:3 for max FOV,
+     * then the highest image resolution) makes preview == capture by
+     * construction -- and usually raises the stitched-frame / photo
+     * resolution above 640x480 as a bonus.
+     *
+     * Must be called on a freshly-created, un-resumed session (ARCore
+     * requires the session paused for setCameraConfig).  Best-effort: on
+     * any failure we keep ARCore's default config.
+     */
+    private fun selectMatchingCameraConfig(session: Session) {
+        try {
+            val configs = session.getSupportedCameraConfigs(CameraConfigFilter(session))
+            if (configs.isEmpty()) return
+            fun aspect(s: android.util.Size): Float = s.width.toFloat() / s.height.toFloat()
+
+            val matched = configs.filter {
+                kotlin.math.abs(aspect(it.imageSize) - aspect(it.textureSize)) < 0.02f
+            }
+            val pool = if (matched.isNotEmpty()) matched else configs
+            val chosen = pool.sortedWith(
+                compareBy<CameraConfig> { kotlin.math.abs(aspect(it.imageSize) - 4f / 3f) }
+                    .thenByDescending { it.imageSize.width * it.imageSize.height },
+            ).firstOrNull() ?: return
+            session.setCameraConfig(chosen)
+            Log.i(
+                TAG,
+                "selectMatchingCameraConfig: chose image=" +
+                    "${chosen.imageSize.width}x${chosen.imageSize.height} texture=" +
+                    "${chosen.textureSize.width}x${chosen.textureSize.height} " +
+                    "(from ${configs.size} configs, ${matched.size} aspect-matched)",
+            )
+        } catch (t: Throwable) {
+            Log.w(TAG, "selectMatchingCameraConfig failed; keeping default config: ${t.message}")
+        }
+    }
+
     companion object {
         // Mirrors RNSARTrackingState on iOS for cross-platform
         // identical JS behaviour.