react-native-image-stitcher 0.1.0

package/android/src/main/java/io/imagestitcher/rn/ar/BackgroundRenderer.kt

@@ -0,0 +1,176 @@
+// SPDX-License-Identifier: Apache-2.0
+package io.imagestitcher.rn.ar
+
+import android.opengl.GLES11Ext
+import android.opengl.GLES20
+import com.google.ar.core.Coordinates2d
+import com.google.ar.core.Frame
+import com.google.ar.core.Session
+import java.nio.ByteBuffer
+import java.nio.ByteOrder
+import java.nio.FloatBuffer
+
+/**
+ * Renders the ARCore camera feed as a fullscreen background quad.
+ *
+ * ARCore delivers camera frames via a GL_TEXTURE_EXTERNAL_OES texture
+ * (the OpenGL ES extension that lets a SurfaceTexture be sampled
+ * directly).  We bind that texture to the session via
+ * `Session.setCameraTextureName()` and render it through the
+ * fullscreen quad below; ARCore handles the camera pixel updates
+ * inside `Session.update()`.
+ *
+ * Why hand-rolled vs adopting the Google hello_ar sample:
+ *   The sample is ~250 lines per renderer plus shader files in
+ *   `assets/shaders/`.  We only need a fullscreen quad — vertex
+ *   coords (-1..1) plus texture coords supplied by ARCore via
+ *   `Frame.transformCoordinates2d()`.  That fits in ~150 lines
+ *   inline, no asset dependency, easier to debug.
+ */
+internal class BackgroundRenderer {
+
+    /// OES texture id ARCore writes camera pixels into.
+    var textureId: Int = -1
+        private set
+
+    /// Compiled shader program.
+    private var program: Int = 0
+    /// Attribute / uniform locations cached at link time.
+    private var positionAttrib: Int = 0
+    private var texCoordAttrib: Int = 0
+    private var textureUniform: Int = 0
+
+    /// Static fullscreen quad coords.
+    private val quadCoords: FloatBuffer = ByteBuffer
+        .allocateDirect(QUAD_COORDS.size * 4)
+        .order(ByteOrder.nativeOrder())
+        .asFloatBuffer()
+        .apply {
+            put(QUAD_COORDS)
+            position(0)
+        }
+
+    /// Texture coords ARCore overwrites each frame via
+    /// `Frame.transformCoordinates2d()`.  The buffer is allocated
+    /// once and refilled in-place to avoid GC pressure.
+    private val quadTexCoords: FloatBuffer = ByteBuffer
+        .allocateDirect(QUAD_COORDS.size * 4)
+        .order(ByteOrder.nativeOrder())
+        .asFloatBuffer()
+
+    /// Has `Session.setCameraTextureName()` been called for the
+    /// current GL context?  `BackgroundRenderer` is created on the
+    /// GL render thread but the Session may already exist; we
+    /// connect them on the first draw call.
+    private var sessionTextureBound: Boolean = false
+
+    fun createOnGlThread() {
+        // Create OES texture for ARCore's camera output.
+        val textures = IntArray(1)
+        GLES20.glGenTextures(1, textures, 0)
+        textureId = textures[0]
+        val target = GLES11Ext.GL_TEXTURE_EXTERNAL_OES
+        GLES20.glBindTexture(target, textureId)
+        GLES20.glTexParameteri(target, GLES20.GL_TEXTURE_WRAP_S, GLES20.GL_CLAMP_TO_EDGE)
+        GLES20.glTexParameteri(target, GLES20.GL_TEXTURE_WRAP_T, GLES20.GL_CLAMP_TO_EDGE)
+        GLES20.glTexParameteri(target, GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_LINEAR)
+        GLES20.glTexParameteri(target, GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_LINEAR)
+
+        program = ShaderUtil.loadGLProgram(VERTEX_SHADER, FRAGMENT_SHADER)
+        positionAttrib = GLES20.glGetAttribLocation(program, "a_Position")
+        texCoordAttrib = GLES20.glGetAttribLocation(program, "a_TexCoord")
+        textureUniform = GLES20.glGetUniformLocation(program, "sTexture")
+        ShaderUtil.checkGLError("BackgroundRenderer.createOnGlThread")
+    }
+
+    fun bindToSession(session: Session) {
+        if (textureId == -1) return
+        session.setCameraTextureName(textureId)
+        sessionTextureBound = true
+    }
+
+    /// Draw the camera feed for the given ARFrame.  Must be called
+    /// from `Renderer.onDrawFrame` after `Session.update()`.  No-op
+    /// if the GL program isn't ready yet (very first frame).
+    fun draw(frame: Frame) {
+        if (program == 0) return
+        // Update the tex coords from the ARFrame whenever the display
+        // geometry has changed (rotation, surface resize).  Cheap
+        // (constant 8 floats) so we do it every frame.
+        if (frame.hasDisplayGeometryChanged()) {
+            frame.transformCoordinates2d(
+                Coordinates2d.OPENGL_NORMALIZED_DEVICE_COORDINATES,
+                quadCoords,
+                Coordinates2d.TEXTURE_NORMALIZED,
+                quadTexCoords,
+            )
+        }
+        // First frame after onSurfaceCreated: do an unconditional fill
+        // so we have valid tex coords before geometryChanged ever flips.
+        if (quadTexCoords.position() == 0 && !frame.hasDisplayGeometryChanged()) {
+            frame.transformCoordinates2d(
+                Coordinates2d.OPENGL_NORMALIZED_DEVICE_COORDINATES,
+                quadCoords,
+                Coordinates2d.TEXTURE_NORMALIZED,
+                quadTexCoords,
+            )
+        }
+        quadCoords.position(0)
+        quadTexCoords.position(0)
+
+        GLES20.glDisable(GLES20.GL_DEPTH_TEST)
+        GLES20.glDepthMask(false)
+        GLES20.glActiveTexture(GLES20.GL_TEXTURE0)
+        GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, textureId)
+        GLES20.glUseProgram(program)
+        GLES20.glUniform1i(textureUniform, 0)
+
+        GLES20.glVertexAttribPointer(positionAttrib, 2, GLES20.GL_FLOAT, false, 0, quadCoords)
+        GLES20.glEnableVertexAttribArray(positionAttrib)
+        GLES20.glVertexAttribPointer(texCoordAttrib, 2, GLES20.GL_FLOAT, false, 0, quadTexCoords)
+        GLES20.glEnableVertexAttribArray(texCoordAttrib)
+
+        GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4)
+
+        GLES20.glDisableVertexAttribArray(positionAttrib)
+        GLES20.glDisableVertexAttribArray(texCoordAttrib)
+        GLES20.glDepthMask(true)
+        GLES20.glEnable(GLES20.GL_DEPTH_TEST)
+        ShaderUtil.checkGLError("BackgroundRenderer.draw")
+    }
+
+    companion object {
+        // Fullscreen quad in NDC space.  4 vertices, GL_TRIANGLE_STRIP
+        // order: bottom-left, bottom-right, top-left, top-right.
+        private val QUAD_COORDS = floatArrayOf(
+            -1f, -1f,
+             1f, -1f,
+            -1f,  1f,
+             1f,  1f,
+        )
+
+        // Vertex shader: pass-through.  Tex coords are pre-transformed
+        // into TEXTURE_NORMALIZED space by ARCore so no extra matrix
+        // math is needed.
+        private val VERTEX_SHADER = """
+            attribute vec4 a_Position;
+            attribute vec2 a_TexCoord;
+            varying vec2 v_TexCoord;
+            void main() {
+                gl_Position = a_Position;
+                v_TexCoord = a_TexCoord;
+            }
+        """.trimIndent()
+
+        // Fragment shader: sample the OES external texture and write.
+        private val FRAGMENT_SHADER = """
+            #extension GL_OES_EGL_image_external : require
+            precision mediump float;
+            uniform samplerExternalOES sTexture;
+            varying vec2 v_TexCoord;
+            void main() {
+                gl_FragColor = texture2D(sTexture, v_TexCoord);
+            }
+        """.trimIndent()
+    }
+}

package/android/src/main/java/io/imagestitcher/rn/ar/ShaderUtil.kt

@@ -0,0 +1,67 @@
+// SPDX-License-Identifier: Apache-2.0
+package io.imagestitcher.rn.ar
+
+import android.opengl.GLES20
+
+/**
+ * Minimal GLSL compile + link helper.  Shared by the BackgroundRenderer
+ * (and any future GL stages we add).  Keeps the per-renderer code
+ * focused on what's drawn rather than on GLSL boilerplate.
+ *
+ * No Android Studio or hello_ar dependency — small enough to inline.
+ */
+internal object ShaderUtil {
+
+    /**
+     * Compile a vertex + fragment shader pair and link into a GL
+     * program.  Throws on any compile/link error with the GL log
+     * attached to the message — much more useful than the bare
+     * GL_FALSE return that GLES20 hands you.
+     */
+    fun loadGLProgram(vertexSrc: String, fragmentSrc: String): Int {
+        val vsh = compileShader(GLES20.GL_VERTEX_SHADER, vertexSrc)
+        val fsh = compileShader(GLES20.GL_FRAGMENT_SHADER, fragmentSrc)
+        val program = GLES20.glCreateProgram()
+        GLES20.glAttachShader(program, vsh)
+        GLES20.glAttachShader(program, fsh)
+        GLES20.glLinkProgram(program)
+        val linked = IntArray(1)
+        GLES20.glGetProgramiv(program, GLES20.GL_LINK_STATUS, linked, 0)
+        if (linked[0] == 0) {
+            val log = GLES20.glGetProgramInfoLog(program)
+            GLES20.glDeleteProgram(program)
+            throw RuntimeException("Shader link failed: $log")
+        }
+        // Shaders can be detached + deleted once linked; the program
+        // holds its own reference to the compiled object code.
+        GLES20.glDetachShader(program, vsh)
+        GLES20.glDetachShader(program, fsh)
+        GLES20.glDeleteShader(vsh)
+        GLES20.glDeleteShader(fsh)
+        return program
+    }
+
+    private fun compileShader(type: Int, src: String): Int {
+        val shader = GLES20.glCreateShader(type)
+        GLES20.glShaderSource(shader, src)
+        GLES20.glCompileShader(shader)
+        val compiled = IntArray(1)
+        GLES20.glGetShaderiv(shader, GLES20.GL_COMPILE_STATUS, compiled, 0)
+        if (compiled[0] == 0) {
+            val log = GLES20.glGetShaderInfoLog(shader)
+            GLES20.glDeleteShader(shader)
+            throw RuntimeException("Shader compile failed: $log\nSource:\n$src")
+        }
+        return shader
+    }
+
+    /// Wrap a GLES call with `glGetError` checking — caller passes a
+    /// human-readable label so the panic message tells you which call
+    /// died.  Cheap; we only call it during init / per-frame setup.
+    fun checkGLError(label: String) {
+        val err = GLES20.glGetError()
+        if (err != GLES20.GL_NO_ERROR) {
+            throw RuntimeException("$label: glError 0x${err.toString(16)}")
+        }
+    }
+}

package/android/src/main/java/io/imagestitcher/rn/ar/YuvImageConverter.kt

@@ -0,0 +1,201 @@
+// SPDX-License-Identifier: Apache-2.0
+package io.imagestitcher.rn.ar
+
+import android.graphics.ImageFormat
+import android.graphics.Rect
+import android.graphics.YuvImage
+import android.media.Image
+import android.view.Surface
+import androidx.exifinterface.media.ExifInterface
+import java.io.ByteArrayOutputStream
+import java.io.File
+import java.io.FileOutputStream
+
+/**
+ * Convert an ARCore `Image` (YUV_420_888) to a JPEG file on disk.
+ *
+ * Why JPEG → file → re-decode by OpenCV (slightly wasteful)?
+ *   The incremental engine's existing API (matching iOS') consumes
+ *   image PATHS, not raw planes.  Threading raw YUV through the
+ *   bridge would require a second native ingestion path.  At ~3-5
+ *   Hz of accepted frames, a ~10 ms YUV→JPEG encode is negligible
+ *   next to the ~40 ms per-frame engine work — keeping the surface
+ *   uniform across iOS / Android paths is worth the few-ms cost.
+ *
+ * `Image` ownership: caller MUST `image.close()` after this returns.
+ * We don't close inside because the caller may want to inspect more
+ * fields (timestamp, format) before releasing.
+ */
+internal object YuvImageConverter {
+
+    /// Convert + write JPEG.  Returns the path (no file:// prefix)
+    /// or null on any encode/write error (caller-decides whether to
+    /// log + drop the frame).
+    ///
+    /// 2026-05-15 (B3) — `displayRotation` parameter writes the
+    /// appropriate EXIF orientation tag so RN's Image loader (and
+    /// any other consumer that respects EXIF) displays the JPEG
+    /// upright regardless of how the device was held at capture.
+    ///
+    /// Without this, Android's image loaders display the raw sensor
+    /// pixels (typically landscape) as-is — so a portrait-held
+    /// capture's thumbnail appears sideways.  iOS's image loader
+    /// auto-respects EXIF orientation; Android's doesn't always
+    /// (depends on the loader path).  Setting the tag covers both.
+    ///
+    /// `displayRotation` should be the value returned from
+    /// `WindowManager.defaultDisplay.rotation` at capture time
+    /// (Surface.ROTATION_0/_90/_180/_270).  We assume a typical
+    /// back-camera sensor orientation of 90° — true for all
+    /// devices we ship to (Galaxy A35 verified).  Wire through
+    /// CameraCharacteristics.SENSOR_ORIENTATION in a follow-up
+    /// if we ever encounter a device that differs.
+    ///
+    /// Default `displayRotation = Surface.ROTATION_0` (portrait)
+    /// preserves the previous behaviour for legacy callsites that
+    /// haven't been updated yet.
+    fun encodeToJpeg(
+        image: Image,
+        outputPath: String,
+        jpegQuality: Int = 70,
+        displayRotation: Int = Surface.ROTATION_0,
+    ): String? {
+        if (image.format != ImageFormat.YUV_420_888) return null
+        val nv21 = yuv420toNV21(image) ?: return null
+        val yuvImage = YuvImage(
+            nv21,
+            ImageFormat.NV21,
+            image.width,
+            image.height,
+            null,
+        )
+        val baos = ByteArrayOutputStream()
+        val ok = yuvImage.compressToJpeg(
+            Rect(0, 0, image.width, image.height),
+            jpegQuality.coerceIn(1, 100),
+            baos,
+        )
+        if (!ok) return null
+        try {
+            FileOutputStream(File(outputPath)).use { it.write(baos.toByteArray()) }
+        } catch (e: Throwable) {
+            return null
+        }
+        // Write EXIF orientation tag based on display rotation.
+        // Sensor orientation 90° assumed (back camera).  The math:
+        //   ROTATION_0  (portrait, sensor 90° CW from screen-up)
+        //     → JPEG needs 90° CW to display upright → ROTATE_90 (6)
+        //   ROTATION_90 (landscape-left, sensor aligned with screen)
+        //     → no rotation → NORMAL (1)
+        //   ROTATION_180 (portrait-upside-down)
+        //     → 270° CW → ROTATE_270 (8)
+        //   ROTATION_270 (landscape-right)
+        //     → 180° → ROTATE_180 (3)
+        //
+        // EXIF tag set EVEN when the orientation is normal — keeps
+        // every output JPEG self-describing for downstream
+        // consumers (cv::Stitcher does NOT auto-honour EXIF, see
+        // BatchStitcher.applyExifOrientation; this metadata
+        // exists primarily for the live thumbnail strip + future
+        // RN Image renderers).
+        val exifOrientation = when (displayRotation) {
+            Surface.ROTATION_0   -> ExifInterface.ORIENTATION_ROTATE_90
+            Surface.ROTATION_90  -> ExifInterface.ORIENTATION_NORMAL
+            Surface.ROTATION_180 -> ExifInterface.ORIENTATION_ROTATE_270
+            Surface.ROTATION_270 -> ExifInterface.ORIENTATION_ROTATE_180
+            else                 -> ExifInterface.ORIENTATION_NORMAL
+        }
+        try {
+            val exif = ExifInterface(outputPath)
+            exif.setAttribute(
+                ExifInterface.TAG_ORIENTATION,
+                exifOrientation.toString(),
+            )
+            exif.saveAttributes()
+        } catch (e: Throwable) {
+            // EXIF write failed — JPEG itself is still valid; just
+            // missing the orientation hint.  Caller doesn't need to
+            // know — non-fatal.
+        }
+        return outputPath
+    }
+
+    /**
+     * Pack a YUV_420_888 `Image` into a contiguous NV21 byte array.
+     *
+     * The Image API exposes Y, U, V as three planes, each with its
+     * own row stride and pixel stride.  NV21 expects a single
+     * contiguous buffer with Y plane first, then interleaved VU bytes
+     * after.  The repacking handles row + pixel strides that don't
+     * match the dense layout.
+     */
+    private fun yuv420toNV21(image: Image): ByteArray? {
+        val w = image.width
+        val h = image.height
+        val ySize = w * h
+        val uvSize = w * h / 2
+
+        val nv21 = ByteArray(ySize + uvSize)
+        val planes = image.planes
+        if (planes.size < 3) return null
+
+        // Y plane.
+        val yPlane = planes[0]
+        val yBuf = yPlane.buffer
+        val yRowStride = yPlane.rowStride
+        if (yRowStride == w) {
+            yBuf.get(nv21, 0, ySize)
+        } else {
+            // Row-by-row copy when stride != width.
+            var dstOffset = 0
+            var srcOffset = 0
+            for (row in 0 until h) {
+                yBuf.position(srcOffset)
+                yBuf.get(nv21, dstOffset, w)
+                dstOffset += w
+                srcOffset += yRowStride
+            }
+        }
+
+        // U + V planes.  YUV_420_888 has them subsampled 2:1 so each
+        // covers (w/2) × (h/2).  Pixel stride is 1 (planar) or 2
+        // (semi-planar interleaved).  NV21 requires interleaved VU.
+        val uPlane = planes[1]
+        val vPlane = planes[2]
+        val uBuf = uPlane.buffer
+        val vBuf = vPlane.buffer
+        val uRowStride = uPlane.rowStride
+        val uPixelStride = uPlane.pixelStride
+        val vRowStride = vPlane.rowStride
+        val vPixelStride = vPlane.pixelStride
+
+        // Most camera2 / ARCore implementations on Android already
+        // produce semi-planar interleaved data with pixelStride=2.
+        // In that case Y plane + V plane (offset by 1) form NV21
+        // directly with a single block copy.  Detect + fast-path it.
+        if (uPixelStride == 2 && vPixelStride == 2 &&
+            uRowStride == vRowStride && uRowStride == w) {
+            // The V plane in NV21 layout starts at vBuf's first byte.
+            // Copy the entire V plane (which physically interleaves
+            // with U bytes since pixelStride=2 means consecutive
+            // bytes are V-U-V-U...).
+            val vBytes = vBuf.remaining().coerceAtMost(uvSize)
+            vBuf.get(nv21, ySize, vBytes)
+            return nv21
+        }
+
+        // Slow path — manual interleave.
+        var pos = ySize
+        val rowsUv = h / 2
+        val colsUv = w / 2
+        for (row in 0 until rowsUv) {
+            for (col in 0 until colsUv) {
+                val vIdx = row * vRowStride + col * vPixelStride
+                val uIdx = row * uRowStride + col * uPixelStride
+                nv21[pos++] = vBuf.get(vIdx)
+                nv21[pos++] = uBuf.get(uIdx)
+            }
+        }
+        return nv21
+    }
+}

package/cpp/ar_frame_pose.h

@@ -0,0 +1,63 @@
+// SPDX-License-Identifier: Apache-2.0
+//
+// ar_frame_pose.h — POD struct shared between iOS and Android for AR
+// frame pose data crossing the C++ boundary.
+//
+// Both platforms marshal their native pose representation into this
+// flat struct:
+//   - iOS:    RNSARFramePose (Swift) → KeyframeGateBridge.mm
+//             unmarshals Swift Doubles → C++ floats.
+//   - Android: RNSARFramePose (Kotlin data class) → JNI
+//             unmarshals JVM doubles → C++ floats.
+//
+// Layout MUST stay stable across both platforms.  Field order /
+// padding / size is checked by static_assert in keyframe_gate.cpp.
+//
+// Convention notes:
+//   - tx, ty, tz: camera origin in world coordinates (metres)
+//   - qx, qy, qz, qw: orientation as a unit quaternion in JPL convention
+//     (last-real-part, matching both ARKit's simd_quatf and ARCore's
+//     Pose.getRotationQuaternion())
+//   - fx, fy, cx, cy: pinhole intrinsics (pixels)
+//   - imageWidth, imageHeight: pixel dimensions of the captured frame
+
+#pragma once
+#include <cstdint>
+
+namespace retailens {
+
+struct Pose {
+    // Translation in metres (world frame).
+    float tx;
+    float ty;
+    float tz;
+    // Rotation quaternion (JPL convention: last component is real).
+    float qx;
+    float qy;
+    float qz;
+    float qw;
+    // Pinhole camera intrinsics (pixels).
+    float fx;
+    float fy;
+    float cx;
+    float cy;
+    // Image dimensions (pixels).
+    int32_t imageWidth;
+    int32_t imageHeight;
+};
+
+// 4×4 column-major rotation+translation matrix.  Matches both
+// simd_float4x4 (iOS) and the array returned by ARCore Pose.toMatrix(...)
+// layout: columns laid out contiguously, so columns.0 is m[0..3],
+// columns.1 is m[4..7], etc.  Translation is the last column (m[12..15]).
+//
+// ARKit ARPlaneAnchor convention:
+//   column 0 = plane tangent X (in-plane "right")
+//   column 1 = plane surface normal
+//   column 2 = plane tangent Z (in-plane "up")
+//   column 3 = plane origin
+struct PlaneTransform {
+    float m[16];
+};
+
+} // namespace retailens