react-native-rectangle-doc-scanner 3.238.0 → 3.240.0

package/android/build.gradle

@@ -68,6 +68,9 @@ dependencies {
     // OpenCV for document detection
     implementation 'org.opencv:opencv:4.9.0'
 
+    // ML Kit object detection for live rectangle hints
+    implementation 'com.google.mlkit:object-detection:17.0.1'
+
     // Coroutines for async operations
     implementation 'org.jetbrains.kotlinx:kotlinx-coroutines-core:1.7.3'
     implementation 'org.jetbrains.kotlinx:kotlinx-coroutines-android:1.7.3'

package/android/src/main/kotlin/com/reactnativerectangledocscanner/CameraController.kt

@@ -23,6 +23,10 @@ import android.util.Size
 import android.view.Surface
 import android.view.TextureView
 import androidx.core.content.ContextCompat
+import com.google.mlkit.vision.common.InputImage
+import com.google.mlkit.vision.objects.ObjectDetection
+import com.google.mlkit.vision.objects.defaults.ObjectDetectorOptions
+import org.opencv.core.Point
 import java.io.File
 import java.io.FileOutputStream
 import java.util.concurrent.atomic.AtomicReference
@@ -58,6 +62,12 @@ class CameraController(
 
     private val pendingCapture = AtomicReference<PendingCapture?>()
     private val analysisInFlight = AtomicBoolean(false)
+    private val objectDetector = ObjectDetection.getClient(
+        ObjectDetectorOptions.Builder()
+            .setDetectorMode(ObjectDetectorOptions.STREAM_MODE)
+            .enableMultipleObjects()
+            .build()
+    )
 
     var onFrameAnalyzed: ((Rectangle?, Int, Int) -> Unit)? = null
 
@@ -179,6 +189,7 @@ class CameraController(
 
     fun shutdown() {
         stopCamera()
+        objectDetector.close()
         cameraThread.quitSafely()
         analysisThread.quitSafely()
     }
@@ -352,21 +363,42 @@ class CameraController(
     }
 
     private fun analyzeImage(image: Image) {
-        try {
-            val nv21 = image.toNv21()
-            val rotationDegrees = computeRotationDegrees()
-            val rectangle = DocumentDetector.detectRectangleInYUV(nv21, image.width, image.height, rotationDegrees)
-
-            val frameWidth = if (rotationDegrees == 90 || rotationDegrees == 270) image.height else image.width
-            val frameHeight = if (rotationDegrees == 90 || rotationDegrees == 270) image.width else image.height
-
-            onFrameAnalyzed?.invoke(rectangle, frameWidth, frameHeight)
+        val rotationDegrees = computeRotationDegrees()
+        val inputImage = try {
+            InputImage.fromMediaImage(image, rotationDegrees)
         } catch (e: Exception) {
-            Log.e(TAG, "[CAMERA2] Error analyzing frame", e)
-        } finally {
+            Log.e(TAG, "[CAMERA2] Failed to create InputImage", e)
             image.close()
             analysisInFlight.set(false)
+            return
         }
+
+        objectDetector.process(inputImage)
+            .addOnSuccessListener { objects ->
+                val best = objects.maxByOrNull { obj ->
+                    val box = obj.boundingBox
+                    box.width() * box.height()
+                }
+                val rectangle = best?.boundingBox?.let { box ->
+                    Rectangle(
+                        Point(box.left.toDouble(), box.top.toDouble()),
+                        Point(box.right.toDouble(), box.top.toDouble()),
+                        Point(box.left.toDouble(), box.bottom.toDouble()),
+                        Point(box.right.toDouble(), box.bottom.toDouble())
+                    )
+                }
+
+                val frameWidth = if (rotationDegrees == 90 || rotationDegrees == 270) image.height else image.width
+                val frameHeight = if (rotationDegrees == 90 || rotationDegrees == 270) image.width else image.height
+                onFrameAnalyzed?.invoke(rectangle, frameWidth, frameHeight)
+            }
+            .addOnFailureListener { e ->
+                Log.e(TAG, "[CAMERA2] ML Kit detection failed", e)
+            }
+            .addOnCompleteListener {
+                image.close()
+                analysisInFlight.set(false)
+            }
     }
 
     private fun processCapture(image: Image, pending: PendingCapture) {
@@ -497,48 +529,6 @@ class CameraController(
         return Bitmap.createBitmap(bitmap, 0, 0, bitmap.width, bitmap.height, matrix, true)
     }
 
-    private fun Image.toNv21(): ByteArray {
-        val width = width
-        val height = height
-        val ySize = width * height
-        val uvSize = width * height / 2
-        val nv21 = ByteArray(ySize + uvSize)
-
-        val yBuffer = planes[0].buffer
-        val uBuffer = planes[1].buffer
-        val vBuffer = planes[2].buffer
-
-        val yRowStride = planes[0].rowStride
-        val yPixelStride = planes[0].pixelStride
-        var outputOffset = 0
-        for (row in 0 until height) {
-            var inputOffset = row * yRowStride
-            for (col in 0 until width) {
-                nv21[outputOffset++] = yBuffer.get(inputOffset)
-                inputOffset += yPixelStride
-            }
-        }
-
-        val uvRowStride = planes[1].rowStride
-        val uvPixelStride = planes[1].pixelStride
-        val vRowStride = planes[2].rowStride
-        val vPixelStride = planes[2].pixelStride
-        val uvHeight = height / 2
-        val uvWidth = width / 2
-        for (row in 0 until uvHeight) {
-            var uInputOffset = row * uvRowStride
-            var vInputOffset = row * vRowStride
-            for (col in 0 until uvWidth) {
-                nv21[outputOffset++] = vBuffer.get(vInputOffset)
-                nv21[outputOffset++] = uBuffer.get(uInputOffset)
-                uInputOffset += uvPixelStride
-                vInputOffset += vPixelStride
-            }
-        }
-
-        return nv21
-    }
-
     private fun hasCameraPermission(): Boolean {
         return ContextCompat.checkSelfPermission(context, Manifest.permission.CAMERA) == PackageManager.PERMISSION_GRANTED
     }

package/android/src/main/kotlin/com/reactnativerectangledocscanner/DocumentDetector.kt

@@ -106,6 +106,7 @@ class DocumentDetector {
             val blurredMat = Mat()
             val cannyMat = Mat()
             val morphMat = Mat()
+            val threshMat = Mat()
 
             try {
                 // Convert to grayscale
@@ -124,57 +125,74 @@ class DocumentDetector {
                 Imgproc.morphologyEx(cannyMat, morphMat, Imgproc.MORPH_CLOSE, kernel)
                 kernel.release()
 
-                // Find contours
-                val contours = mutableListOf<MatOfPoint>()
-                val hierarchy = Mat()
-                Imgproc.findContours(
-                    morphMat,
-                    contours,
-                    hierarchy,
-                    Imgproc.RETR_EXTERNAL,
-                    Imgproc.CHAIN_APPROX_SIMPLE
-                )
-
-                // Find the largest contour that approximates to a quadrilateral
-                var largestRectangle: Rectangle? = null
-                var largestArea = 0.0
-                val minArea = max(600.0, (srcMat.rows() * srcMat.cols()) * 0.001)
-
-                for (contour in contours) {
-                    val contourArea = Imgproc.contourArea(contour)
-
-                    // Filter small contours
-                    if (contourArea < minArea) continue
-
-                    // Approximate contour to polygon
-                    val approx = MatOfPoint2f()
-                    val contour2f = MatOfPoint2f(*contour.toArray())
-                    val epsilon = 0.02 * Imgproc.arcLength(contour2f, true)
-                    Imgproc.approxPolyDP(contour2f, approx, epsilon, true)
-
-                    // Check if it's a quadrilateral
-                    if (approx.total() == 4L && Imgproc.isContourConvex(MatOfPoint(*approx.toArray()))) {
-                        val points = approx.toArray()
-
-                        if (contourArea > largestArea) {
-                            largestArea = contourArea
-                            largestRectangle = orderPoints(points)
+                fun findLargestRectangle(source: Mat): Rectangle? {
+                    val contours = mutableListOf<MatOfPoint>()
+                    val hierarchy = Mat()
+                    Imgproc.findContours(
+                        source,
+                        contours,
+                        hierarchy,
+                        Imgproc.RETR_EXTERNAL,
+                        Imgproc.CHAIN_APPROX_SIMPLE
+                    )
+
+                    var largestRectangle: Rectangle? = null
+                    var largestArea = 0.0
+                    val minArea = max(500.0, (srcMat.rows() * srcMat.cols()) * 0.0008)
+
+                    for (contour in contours) {
+                        val contourArea = Imgproc.contourArea(contour)
+                        if (contourArea < minArea) continue
+
+                        val approx = MatOfPoint2f()
+                        val contour2f = MatOfPoint2f(*contour.toArray())
+                        val epsilon = 0.018 * Imgproc.arcLength(contour2f, true)
+                        Imgproc.approxPolyDP(contour2f, approx, epsilon, true)
+
+                        if (approx.total() == 4L && Imgproc.isContourConvex(MatOfPoint(*approx.toArray()))) {
+                            val points = approx.toArray()
+                            if (contourArea > largestArea) {
+                                largestArea = contourArea
+                                largestRectangle = orderPoints(points)
+                            }
                         }
+
+                        approx.release()
+                        contour2f.release()
                     }
 
-                    approx.release()
-                    contour2f.release()
+                    hierarchy.release()
+                    contours.forEach { it.release() }
+                    return largestRectangle
                 }
 
-                hierarchy.release()
-                contours.forEach { it.release() }
+                // First pass: Canny-based edges (good for strong edges).
+                var rectangle = findLargestRectangle(morphMat)
+
+                // Fallback: adaptive threshold (better for low-contrast cards).
+                if (rectangle == null) {
+                    Imgproc.adaptiveThreshold(
+                        blurredMat,
+                        threshMat,
+                        255.0,
+                        Imgproc.ADAPTIVE_THRESH_GAUSSIAN_C,
+                        Imgproc.THRESH_BINARY,
+                        15,
+                        2.0
+                    )
+                    val kernel = Imgproc.getStructuringElement(Imgproc.MORPH_RECT, Size(3.0, 3.0))
+                    Imgproc.morphologyEx(threshMat, morphMat, Imgproc.MORPH_CLOSE, kernel)
+                    kernel.release()
+                    rectangle = findLargestRectangle(morphMat)
+                }
 
-                return largestRectangle
+                return rectangle
             } finally {
                 grayMat.release()
                 blurredMat.release()
                 cannyMat.release()
                 morphMat.release()
+                threshMat.release()
             }
         }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "react-native-rectangle-doc-scanner",
-  "version": "3.238.0",
+  "version": "3.240.0",
   "description": "Native-backed document scanner for React Native with customizable overlays.",
   "license": "MIT",
   "main": "dist/index.js",