react-native-yolo 0.0.3 → 0.0.5

package/android/fix-prefab.gradle

@@ -1,51 +1,51 @@
-tasks.configureEach { task ->
-  // Make sure that we generate our prefab publication file only after having built the native library
-  // so that not a header publication file, but a full configuration publication will be generated, which
-  // will include the .so file
-
-  def prefabConfigurePattern = ~/^prefab(.+)ConfigurePackage$/
-  def matcher = task.name =~ prefabConfigurePattern
-  if (matcher.matches()) {
-    def variantName = matcher[0][1]
-    task.outputs.upToDateWhen { false }
-    task.dependsOn("externalNativeBuild${variantName}")
-  }
-}
-
-afterEvaluate {
-  def abis = reactNativeArchitectures()
-  rootProject.allprojects.each { proj ->
-    if (proj === rootProject) return
-
-    def dependsOnThisLib = proj.configurations.findAll { it.canBeResolved }.any { config ->
-      config.dependencies.any { dep ->
-        dep.group == project.group && dep.name == project.name
-      }
-    }
-    if (!dependsOnThisLib && proj != project) return
-
-    if (!proj.plugins.hasPlugin('com.android.application') && !proj.plugins.hasPlugin('com.android.library')) {
-      return
-    }
-
-    def variants = proj.android.hasProperty('applicationVariants') ? proj.android.applicationVariants : proj.android.libraryVariants
-    // Touch the prefab_config.json files to ensure that in ExternalNativeJsonGenerator.kt we will re-trigger the prefab CLI to
-    // generate a libnameConfig.cmake file that will contain our native library (.so).
-    // See this condition: https://cs.android.com/android-studio/platform/tools/base/+/mirror-goog-studio-main:build-system/gradle-core/src/main/java/com/android/build/gradle/tasks/ExternalNativeJsonGenerator.kt;l=207-219?q=createPrefabBuildSystemGlue
-    variants.all { variant ->
-      def variantName = variant.name
-      abis.each { abi ->
-        def searchDir = new File(proj.projectDir, ".cxx/${variantName}")
-        if (!searchDir.exists()) return
-        def matches = []
-        searchDir.eachDir { randomDir ->
-          def prefabFile = new File(randomDir, "${abi}/prefab_config.json")
-          if (prefabFile.exists()) matches << prefabFile
-        }
-        matches.each { prefabConfig ->
-          prefabConfig.setLastModified(System.currentTimeMillis())
-        }
-      }
-    }
-  }
+tasks.configureEach { task ->
+  // Make sure that we generate our prefab publication file only after having built the native library
+  // so that not a header publication file, but a full configuration publication will be generated, which
+  // will include the .so file
+
+  def prefabConfigurePattern = ~/^prefab(.+)ConfigurePackage$/
+  def matcher = task.name =~ prefabConfigurePattern
+  if (matcher.matches()) {
+    def variantName = matcher[0][1]
+    task.outputs.upToDateWhen { false }
+    task.dependsOn("externalNativeBuild${variantName}")
+  }
+}
+
+afterEvaluate {
+  def abis = reactNativeArchitectures()
+  rootProject.allprojects.each { proj ->
+    if (proj === rootProject) return
+
+    def dependsOnThisLib = proj.configurations.findAll { it.canBeResolved }.any { config ->
+      config.dependencies.any { dep ->
+        dep.group == project.group && dep.name == project.name
+      }
+    }
+    if (!dependsOnThisLib && proj != project) return
+
+    if (!proj.plugins.hasPlugin('com.android.application') && !proj.plugins.hasPlugin('com.android.library')) {
+      return
+    }
+
+    def variants = proj.android.hasProperty('applicationVariants') ? proj.android.applicationVariants : proj.android.libraryVariants
+    // Touch the prefab_config.json files to ensure that in ExternalNativeJsonGenerator.kt we will re-trigger the prefab CLI to
+    // generate a libnameConfig.cmake file that will contain our native library (.so).
+    // See this condition: https://cs.android.com/android-studio/platform/tools/base/+/mirror-goog-studio-main:build-system/gradle-core/src/main/java/com/android/build/gradle/tasks/ExternalNativeJsonGenerator.kt;l=207-219?q=createPrefabBuildSystemGlue
+    variants.all { variant ->
+      def variantName = variant.name
+      abis.each { abi ->
+        def searchDir = new File(proj.projectDir, ".cxx/${variantName}")
+        if (!searchDir.exists()) return
+        def matches = []
+        searchDir.eachDir { randomDir ->
+          def prefabFile = new File(randomDir, "${abi}/prefab_config.json")
+          if (prefabFile.exists()) matches << prefabFile
+        }
+        matches.each { prefabConfig ->
+          prefabConfig.setLastModified(System.currentTimeMillis())
+        }
+      }
+    }
+  }
 }

package/android/gradle.properties

@@ -1,5 +1,5 @@
-Yolo_kotlinVersion=2.1.20
-Yolo_minSdkVersion=23
-Yolo_targetSdkVersion=35
-Yolo_compileSdkVersion=34
-Yolo_ndkVersion=27.1.12297006
+Yolo_kotlinVersion=2.1.20
+Yolo_minSdkVersion=23
+Yolo_targetSdkVersion=35
+Yolo_compileSdkVersion=34
+Yolo_ndkVersion=27.1.12297006

package/android/src/main/AndroidManifest.xml

@@ -1,2 +1,3 @@
-<manifest xmlns:android="http://schemas.android.com/apk/res/android">
-</manifest>
+<manifest xmlns:android="http://schemas.android.com/apk/res/android">
+    <uses-feature android:name="android.hardware.camera" android:required="false" />
+</manifest>

package/android/src/main/cpp/cpp-adapter.cpp

@@ -1,9 +1,9 @@
-#include <jni.h>
-#include <fbjni/fbjni.h>
-#include "YoloOnLoad.hpp"
-
-JNIEXPORT jint JNICALL JNI_OnLoad(JavaVM* vm, void*) {
-  return facebook::jni::initialize(vm, []() {
-    margelo::nitro::yolo::registerAllNatives();
-  });
+#include <jni.h>
+#include <fbjni/fbjni.h>
+#include "YoloOnLoad.hpp"
+
+JNIEXPORT jint JNICALL JNI_OnLoad(JavaVM* vm, void*) {
+  return facebook::jni::initialize(vm, []() {
+    margelo::nitro::yolo::registerAllNatives();
+  });
 }

package/android/src/main/java/com/yolo/HybridYolo.kt

@@ -1,42 +1,265 @@
-package com.yolo
-
-import android.net.Uri
-import android.util.Log
-import com.margelo.nitro.NitroModules
-import com.margelo.nitro.yolo.HybridYoloSpec
-import java.io.File
-import java.io.RandomAccessFile
-import java.net.URL
-import java.nio.MappedByteBuffer
-import java.nio.channels.FileChannel
-import org.tensorflow.lite.Interpreter
-import org.tensorflow.lite.support.common.FileUtil
-import yolo.com.loader.YoloModelLoader
-
-class HybridYolo : HybridYoloSpec() {
-    private var interpreter: Interpreter? = null
-    private val modelLoader = YoloModelLoader()
-
-    override fun sum(num1: Double, num2: Double): Double {
-        return num1 + num2
-    }
-
-    override fun loadModel(modelPath: String) {
-        val context =
-                NitroModules.applicationContext ?: throw IllegalStateException("Context is null")
-
-        Log.d("YOLO_TAG", "Trying to load: $modelPath")
-
-        try {
-            val modelBuffer = modelLoader.load(modelPath)
-
-            interpreter?.close()
-            interpreter = Interpreter(modelBuffer)
-
-            Log.d("YOLO_TAG", "✅ Model loaded successfully!")
-        } catch (e: Exception) {
-            Log.e("YOLO_TAG", "❌ Failed to load model: ${e.message}", e)
-        }
-    }
-
-}
+package com.yolo
+
+import android.net.Uri
+import android.util.Log
+import com.margelo.nitro.NitroModules
+import com.margelo.nitro.yolo.HybridYoloSpec
+import java.io.File
+import java.io.RandomAccessFile
+import java.net.URL
+import java.nio.MappedByteBuffer
+import java.nio.channels.FileChannel
+import org.tensorflow.lite.Interpreter
+import org.tensorflow.lite.support.common.FileUtil
+import yolo.com.loader.YoloModelLoader
+import com.margelo.nitro.camera.HybridFrameSpec
+
+import android.util.Base64
+import com.yolo.utils.FrameJpegConverter
+import com.yolo.utils.FrameValidator
+
+import com.margelo.nitro.yolo.Detection
+import com.margelo.nitro.yolo.BoundingBox
+
+import java.nio.ByteBuffer
+import java.nio.ByteOrder
+import org.tensorflow.lite.DataType
+import kotlin.math.roundToInt
+
+
+class HybridYolo : HybridYoloSpec() {
+    companion object {
+        private const val TAG = "YOLO_TAG"
+    }
+    private var interpreter: Interpreter? = null
+
+    private var inputBuffer: ByteBuffer? = null
+    private var inputWidth = 0
+    private var inputHeight = 0
+    private var inputDataType: DataType? = null
+
+    private val modelLoader = YoloModelLoader()
+
+    override fun sum(num1: Double, num2: Double): Double {
+        return num1 + num2
+    }
+
+    override fun loadModel(modelPath: String) {
+        val context =
+                NitroModules.applicationContext ?: throw IllegalStateException("Context is null")
+
+        Log.d(TAG, "Trying to load: $modelPath")
+
+        try {
+            val modelBuffer = modelLoader.load(modelPath)
+
+            interpreter?.close()
+            interpreter = Interpreter(modelBuffer)
+
+            val inputTensor = interpreter!!.getInputTensor(0)
+            val shape = inputTensor.shape()
+
+            inputHeight = shape[1]
+            inputWidth = shape[2]
+            inputDataType = inputTensor.dataType()
+            inputBuffer = modelLoader.makeInputBuffer(interpreter!!)
+
+            val outputTensor = interpreter!!.getOutputTensor(0)
+            Log.d(TAG, "✅ YOLO model loaded")
+            Log.d(TAG, "📥 Input shape: ${inputTensor.shape().contentToString()}")
+            Log.d(TAG, "📤 Output shape: ${outputTensor.shape().contentToString()}")
+            Log.d(TAG, "📥 Input type: ${inputTensor.dataType()}")
+            Log.d(TAG, "📤 Output type: ${outputTensor.dataType()}")
+
+            Log.d(TAG, "Input shape=${shape.contentToString()} type=$inputDataType")
+            Log.d(TAG, "✅ Model loaded successfully!")
+        } catch (e: Exception) {
+            Log.e(TAG, "❌ Failed to load model: ${e.message}", e)
+        }
+    }
+    override fun frameToBase64(frame: HybridFrameSpec): String {
+        return try {
+            if (!FrameValidator.isValidYuv(frame)) return ""
+
+            val jpegBytes = FrameJpegConverter.toJpegBytes(
+            frame = frame,
+            quality = 80
+            )
+
+            Base64.encodeToString(jpegBytes, Base64.NO_WRAP)
+        } catch (e: Exception) {
+            Log.e(TAG, "❌ frameToBase64 failed", e)
+            ""
+        }
+    }
+
+    override fun detect(frame: HybridFrameSpec): Array<Detection> {
+        val localInterpreter = interpreter ?: run {
+            Log.e(TAG, "❌ Model is not loaded. Please call loadModel() first.")
+            return emptyArray()
+        }
+
+        if (!FrameValidator.isValidYuv(frame)) {
+            Log.e(TAG, "❌ Invalid frame provided for detection.")
+            return emptyArray()
+        }
+
+        val input = inputBuffer ?: return emptyArray()
+
+        fillInputFromYuvFrame(
+            frame = frame,
+            input = input,
+            dstWidth = inputWidth,
+            dstHeight = inputHeight,
+            dataType = inputDataType ?: DataType.FLOAT32
+        )
+
+        val output = Array(1) { Array(300) { FloatArray(6) } }
+
+        localInterpreter.run(input, output)
+
+        val detections = parseNmsOutput(output, confidenceThreshold = 0.5f)
+        if (detections.isEmpty()) {
+            val best = output[0].maxByOrNull { it[4] }
+            Log.d(TAG, "No detections. Best row=${best?.contentToString()}")
+        } else {
+            Log.d(TAG, "✅ Detections: ${detections.size}")
+        }
+
+
+        return detections.toTypedArray()
+    }
+
+    private fun parseNmsOutput(
+        output: Array<Array<FloatArray>>,
+        confidenceThreshold: Float = 0.5f
+    ): List<Detection> {
+        val detections = mutableListOf<Detection>()
+
+        val batchMatrix = output[0]
+
+        for (i in batchMatrix.indices) {
+            val row = batchMatrix[i]
+
+            val score = row[4]
+            if (score < confidenceThreshold) continue
+
+            val x1 = row[0].toDouble()
+            val y1 = row[1].toDouble()
+            val x2 = row[2].toDouble()
+            val y2 = row[3].toDouble()
+            val classId = row[5].toDouble()
+
+            detections.add(
+                Detection(
+                    boundingBox = BoundingBox(
+                        x1 = x1,
+                        y1 = y1,
+                        x2 = x2,
+                        y2 = y2
+                    ),
+                    score = score.toDouble(),
+                    classId = classId
+                )
+            )
+        }
+
+        return detections
+    }
+
+
+
+    private fun fillInputFromYuvFrame(
+        frame: HybridFrameSpec,
+        input: ByteBuffer,
+        dstWidth: Int,
+        dstHeight: Int,
+        dataType: DataType
+    ) {
+        val srcWidth = frame.width.toInt()
+        val srcHeight = frame.height.toInt()
+
+        val planes = frame.getPlanes()
+
+        val yPlane = planes[0]
+        val uPlane = planes[1]
+        val vPlane = planes[2]
+
+        val yBytes = yPlane.getPixelBuffer().toByteArray()
+        val uBytes = uPlane.getPixelBuffer().toByteArray()
+        val vBytes = vPlane.getPixelBuffer().toByteArray()
+
+        val yRowStride = yPlane.bytesPerRow.roundToInt()
+        val uRowStride = uPlane.bytesPerRow.roundToInt()
+        val vRowStride = vPlane.bytesPerRow.roundToInt()
+
+        input.rewind()
+
+        for (dy in 0 until dstHeight) {
+            // val sy = dy * srcHeight / dstHeight
+
+            for (dx in 0 until dstWidth) {
+                val srcX = dy * srcWidth / dstHeight
+                val srcY = srcHeight - 1 - (dx * srcHeight / dstWidth)
+
+                val yIndex = srcY * yRowStride + srcX
+
+                val uvX = srcX / 2
+                val uvY = srcY / 2
+
+                val uIndex = uvY * uRowStride + uvX * 2
+                val vIndex = uvY * vRowStride + uvX * 2
+
+                if (
+                    yIndex >= yBytes.size ||
+                    uIndex >= uBytes.size ||
+                    vIndex >= vBytes.size
+                ) {
+                    continue
+                }
+
+                val y = yBytes[yIndex].toInt() and 0xFF
+                val u = uBytes[uIndex].toInt() and 0xFF
+                val v = vBytes[vIndex].toInt() and 0xFF
+
+                val rFloat = y + 1.402f * (v - 128)
+                val gFloat = y - 0.344136f * (u - 128) - 0.714136f * (v - 128)
+                val bFloat = y + 1.772f * (u - 128)
+
+                val r = rFloat.roundToInt().coerceIn(0, 255)
+                val g = gFloat.roundToInt().coerceIn(0, 255)
+                val b = bFloat.roundToInt().coerceIn(0, 255)
+
+                when (dataType) {
+                    DataType.FLOAT32 -> {
+                        input.putFloat(r / 255f)
+                        input.putFloat(g / 255f)
+                        input.putFloat(b / 255f)
+                    }
+
+                    DataType.UINT8 -> {
+                        input.put(r.toByte())
+                        input.put(g.toByte())
+                        input.put(b.toByte())
+                    }
+
+                    else -> error("Unsupported input type: $dataType")
+                }
+            }
+        }
+        input.rewind()
+    }
+
+    private fun yuvToRgb(y: Int, u: Int, v: Int): IntArray {
+        val yf = y.toFloat()
+        val uf = u.toFloat() - 128f
+        val vf = v.toFloat() - 128f
+
+        val r = (yf + 1.402f * vf).roundToInt().coerceIn(0, 255)
+        val g = (yf - 0.344136f * uf - 0.714136f * vf).roundToInt().coerceIn(0, 255)
+        val b = (yf + 1.772f * uf).roundToInt().coerceIn(0, 255)
+
+        return intArrayOf(r, g, b)
+    }
+
+}

package/android/src/main/java/com/yolo/YoloPackage.kt

@@ -1,20 +1,20 @@
-package com.yolo;
-
-import com.facebook.react.bridge.NativeModule;
-import com.facebook.react.bridge.ReactApplicationContext;
-import com.facebook.react.module.model.ReactModuleInfoProvider;
-import com.facebook.react.BaseReactPackage;
-import com.margelo.nitro.yolo.YoloOnLoad;
-
-
-public class YoloPackage : BaseReactPackage() {
-  override fun getModule(name: String, reactContext: ReactApplicationContext): NativeModule? = null
-
-  override fun getReactModuleInfoProvider(): ReactModuleInfoProvider = ReactModuleInfoProvider { emptyMap() }
-
-  companion object {
-    init {
-      YoloOnLoad.initializeNative();
-    }
-  }
-}
+package com.yolo;
+
+import com.facebook.react.bridge.NativeModule;
+import com.facebook.react.bridge.ReactApplicationContext;
+import com.facebook.react.module.model.ReactModuleInfoProvider;
+import com.facebook.react.BaseReactPackage;
+import com.margelo.nitro.yolo.YoloOnLoad;
+
+
+public class YoloPackage : BaseReactPackage() {
+  override fun getModule(name: String, reactContext: ReactApplicationContext): NativeModule? = null
+
+  override fun getReactModuleInfoProvider(): ReactModuleInfoProvider = ReactModuleInfoProvider { emptyMap() }
+
+  companion object {
+    init {
+      YoloOnLoad.initializeNative();
+    }
+  }
+}

package/android/src/main/java/com/yolo/loader/YoloModelLoader.kt

@@ -10,6 +10,12 @@ import java.nio.MappedByteBuffer
 import java.nio.channels.FileChannel
 import org.tensorflow.lite.support.common.FileUtil
 
+import org.tensorflow.lite.DataType
+import java.nio.ByteBuffer
+import java.nio.ByteOrder
+import kotlin.math.roundToInt
+import org.tensorflow.lite.Interpreter
+
 
 
 /**
@@ -127,4 +133,29 @@ class YoloModelLoader {
 
         return file
     }
+
+
+    fun makeInputBuffer(interpreter: Interpreter): ByteBuffer {
+        val inputTensor = interpreter.getInputTensor(0)
+        val shape = inputTensor.shape() // usually [1, 640, 640, 3]
+        val dataType = inputTensor.dataType()
+
+        val batch = shape[0]
+        val height = shape[1]
+        val width = shape[2]
+        val channels = shape[3]
+
+        require(batch == 1)
+        require(channels == 3)
+
+        val bytesPerValue = when (dataType) {
+            DataType.FLOAT32 -> 4
+            DataType.UINT8 -> 1
+            else -> error("Unsupported input type: $dataType")
+        }
+
+        return ByteBuffer
+            .allocateDirect(batch * width * height * channels * bytesPerValue)
+            .order(ByteOrder.nativeOrder())
+    }
 }

package/android/src/main/java/com/yolo/utils/BitmapOrientationFixer.kt

@@ -0,0 +1,56 @@
+package com.yolo.utils
+
+import android.graphics.Bitmap
+import android.graphics.BitmapFactory
+import android.graphics.Matrix
+import com.margelo.nitro.camera.HybridFrameSpec
+import java.io.ByteArrayOutputStream
+
+
+
+object BitmapOrientationFixer {
+  fun fix(
+    jpegBytes: ByteArray,
+    frame: HybridFrameSpec,
+    quality: Int
+  ): ByteArray {
+    val bitmap = BitmapFactory.decodeByteArray(
+      jpegBytes,
+      0,
+      jpegBytes.size
+    ) ?: return jpegBytes
+
+    val rotationDegrees = 90f //TODO: Get the actual rotation degrees from the frame metadata if available
+
+    val matrix = Matrix().apply {
+      postRotate(rotationDegrees)
+
+      if (frame.isMirrored) {
+        postScale(-1f, 1f)
+      }
+    }
+
+    val rotatedBitmap = Bitmap.createBitmap(
+      bitmap,
+      0,
+      0,
+      bitmap.width,
+      bitmap.height,
+      matrix,
+      true
+    )
+
+    val output = ByteArrayOutputStream()
+
+    rotatedBitmap.compress(
+      Bitmap.CompressFormat.JPEG,
+      quality,
+      output
+    )
+
+    bitmap.recycle()
+    rotatedBitmap.recycle()
+
+    return output.toByteArray()
+  }
+}

package/android/src/main/java/com/yolo/utils/FrameJpegConverter.kt

@@ -0,0 +1,27 @@
+package com.yolo.utils
+
+import android.util.Log
+import com.margelo.nitro.camera.HybridFrameSpec
+import kotlin.math.roundToInt
+
+object FrameJpegConverter {
+    private const val TAG = "YOLO_TAG_FrameJpegConverter"
+    fun toJpegBytes(frame : HybridFrameSpec, quality: Int = 80): ByteArray {
+        val width = frame.width.roundToInt()
+        val height = frame.height.roundToInt()
+
+        val nv21 = Yuv420ToNv21Converter.convert(frame, width, height)
+
+        val jpegBytes = Nv21JpegEncoder.encode(
+            nv21 = nv21,
+            width = width,
+            height = height,
+            quality = quality
+        )
+        return BitmapOrientationFixer.fix(
+            jpegBytes = jpegBytes,
+            frame = frame,
+            quality = quality
+        )
+    }
+}

package/android/src/main/java/com/yolo/utils/FrameValidator.kt

@@ -0,0 +1,29 @@
+package com.yolo.utils
+
+import android.util.Log
+import com.margelo.nitro.camera.HybridFrameSpec
+
+
+object FrameValidator {
+    private const val TAG = "YOLO_TAG_FrameValidator"
+    fun isValidYuv(frame: HybridFrameSpec): Boolean {
+        if (!frame.isValid) {
+            Log.e(TAG, "❌ Frame is not valid")
+            return false
+        }
+
+        val planes = frame.getPlanes()
+        if (planes.size < 3) {
+            Log.e(TAG, "❌ Expected 3 YUV planes, got ${planes.size}")
+            return false
+        }
+
+        planes.forEachIndexed { index, plane ->
+            if (!plane.isValid) {
+                Log.e(TAG, "❌ Plane $index is not valid")
+                return false
+            }
+        }
+        return true
+    }
+}

package/android/src/main/java/com/yolo/utils/Nv21JpegEncoder.kt

@@ -0,0 +1,34 @@
+package com.yolo.utils
+
+import android.graphics.ImageFormat
+import android.graphics.Rect
+import android.graphics.YuvImage
+import java.io.ByteArrayOutputStream
+
+
+object Nv21JpegEncoder {
+    fun encode(
+        nv21: ByteArray,
+        width: Int,
+        height: Int,
+        quality: Int
+    ): ByteArray {
+        val yuvImage = YuvImage(
+        nv21,
+        ImageFormat.NV21,
+        width,
+        height,
+        null
+        )
+
+        val output = ByteArrayOutputStream()
+
+        yuvImage.compressToJpeg(
+        Rect(0, 0, width, height),
+        quality,
+        output
+        )
+
+        return output.toByteArray()
+    }
+}