react-native-executorch 0.4.8 → 0.5.1-rc.0

package/android/src/main/java/com/swmansion/rnexecutorch/utils/ArrayUtils.kt

@@ -1,87 +0,0 @@
-package com.swmansion.rnexecutorch.utils
-
-import com.facebook.react.bridge.Arguments
-import com.facebook.react.bridge.ReadableArray
-import com.facebook.react.bridge.WritableArray
-import org.pytorch.executorch.DType
-import org.pytorch.executorch.EValue
-import org.pytorch.executorch.Tensor
-
-class ArrayUtils {
-  companion object {
-    inline fun <reified T> createTypedArrayFromReadableArray(
-      input: ReadableArray,
-      transform: (ReadableArray, Int) -> T,
-    ): Array<T> = Array(input.size()) { index -> transform(input, index) }
-
-    fun createByteArray(input: ReadableArray): ByteArray =
-      createTypedArrayFromReadableArray(input) { array, index -> array.getInt(index).toByte() }.toByteArray()
-
-    fun createCharArray(input: ReadableArray): CharArray =
-      createTypedArrayFromReadableArray(input) { array, index -> array.getInt(index).toChar() }.toCharArray()
-
-    fun createIntArray(input: ReadableArray): IntArray =
-      createTypedArrayFromReadableArray(input) { array, index -> array.getInt(index) }.toIntArray()
-
-    fun createFloatArray(input: ReadableArray): FloatArray =
-      createTypedArrayFromReadableArray(input) { array, index -> array.getDouble(index).toFloat() }.toFloatArray()
-
-    fun createLongArray(input: ReadableArray): LongArray =
-      createTypedArrayFromReadableArray(input) { array, index -> array.getInt(index).toLong() }.toLongArray()
-
-    fun createDoubleArray(input: ReadableArray): DoubleArray =
-      createTypedArrayFromReadableArray(input) { array, index -> array.getDouble(index) }.toDoubleArray()
-
-    fun createReadableArrayFromTensor(result: Tensor): ReadableArray {
-      val resultArray = Arguments.createArray()
-
-      when (result.dtype()) {
-        DType.UINT8 -> {
-          result.dataAsByteArray.forEach { resultArray.pushInt(it.toInt()) }
-        }
-
-        DType.INT32 -> {
-          result.dataAsIntArray.forEach { resultArray.pushInt(it) }
-        }
-
-        DType.FLOAT -> {
-          result.dataAsFloatArray.forEach { resultArray.pushDouble(it.toDouble()) }
-        }
-
-        DType.DOUBLE -> {
-          result.dataAsDoubleArray.forEach { resultArray.pushDouble(it) }
-        }
-
-        DType.INT64 -> {
-          // TODO: Do something to handle or deprecate long dtype
-          // https://github.com/facebook/react-native/issues/12506
-          result.dataAsLongArray.forEach { resultArray.pushInt(it.toInt()) }
-        }
-
-        else -> {
-          throw IllegalArgumentException("Invalid dtype: ${result.dtype()}")
-        }
-      }
-
-      return resultArray
-    }
-
-    fun createReadableArrayFromFloatArray(input: FloatArray): ReadableArray {
-      val resultArray = Arguments.createArray()
-      input.forEach { resultArray.pushDouble(it.toDouble()) }
-      return resultArray
-    }
-
-    fun createReadableArrayFromIntArray(input: IntArray): ReadableArray {
-      val resultArray = Arguments.createArray()
-      input.forEach { resultArray.pushInt(it) }
-      return resultArray
-    }
-
-    fun writableArrayToEValue(input: WritableArray): EValue {
-      val size = input.size()
-      val preprocessorInputShape = longArrayOf(1, size.toLong())
-      return EValue.from(Tensor.fromBlob(createFloatArray(input), preprocessorInputShape))
-    }
-  }
-}

package/android/src/main/java/com/swmansion/rnexecutorch/utils/ETError.kt

@@ -1,34 +0,0 @@
-package com.swmansion.rnexecutorch.utils
-
-enum class ETError(
-  val code: Int,
-) {
-  UndefinedError(0x65),
-  ModuleNotLoaded(0x66),
-  FileWriteFailed(0x67),
-  InvalidModelSource(0xff),
-
-  // System errors
-  Ok(0x00),
-  Internal(0x01),
-  InvalidState(0x02),
-  EndOfMethod(0x03),
-
-  // Logical errors
-  NotSupported(0x10),
-  NotImplemented(0x11),
-  InvalidArgument(0x12),
-  InvalidType(0x13),
-  OperatorMissing(0x14),
-
-  // Resource errors
-  NotFound(0x20),
-  MemoryAllocationFailed(0x21),
-  AccessFailed(0x22),
-  InvalidProgram(0x23),
-
-  // Delegate errors
-  DelegateInvalidCompatibility(0x30),
-  DelegateMemoryAllocationFailed(0x31),
-  DelegateInvalidHandle(0x32),
-}

package/android/src/main/java/com/swmansion/rnexecutorch/utils/ImageProcessor.kt

@@ -1,237 +0,0 @@
-package com.swmansion.rnexecutorch.utils
-
-import android.content.Context
-import android.net.Uri
-import android.util.Base64
-import org.opencv.core.Core
-import org.opencv.core.CvType
-import org.opencv.core.Mat
-import org.opencv.core.Scalar
-import org.opencv.core.Size
-import org.opencv.imgcodecs.Imgcodecs
-import org.opencv.imgproc.Imgproc
-import org.pytorch.executorch.EValue
-import org.pytorch.executorch.Tensor
-import java.io.File
-import java.io.InputStream
-import java.net.URL
-import java.util.UUID
-import kotlin.math.floor
-
-class ImageProcessor {
-  companion object {
-    fun matToEValue(
-      mat: Mat,
-      shape: LongArray,
-    ): EValue = matToEValue(mat, shape, Scalar(0.0, 0.0, 0.0), Scalar(1.0, 1.0, 1.0))
-
-    fun matToEValue(
-      mat: Mat,
-      shape: LongArray,
-      mean: Scalar,
-      variance: Scalar,
-    ): EValue {
-      val pixelCount = mat.cols() * mat.rows()
-      val floatArray = FloatArray(pixelCount * 3)
-
-      for (i in 0 until pixelCount) {
-        val row = i / mat.cols()
-        val col = i % mat.cols()
-        val pixel = mat.get(row, col)
-
-        if (mat.type() == CvType.CV_8UC3 || mat.type() == CvType.CV_8UC4) {
-          val b = (pixel[0] - mean.`val`[0] * 255.0f) / (variance.`val`[0] * 255.0f)
-          val g = (pixel[1] - mean.`val`[1] * 255.0f) / (variance.`val`[1] * 255.0f)
-          val r = (pixel[2] - mean.`val`[2] * 255.0f) / (variance.`val`[2] * 255.0f)
-
-          floatArray[0 * pixelCount + i] = b.toFloat()
-          floatArray[1 * pixelCount + i] = g.toFloat()
-          floatArray[2 * pixelCount + i] = r.toFloat()
-        }
-      }
-
-      return EValue.from(Tensor.fromBlob(floatArray, shape))
-    }
-
-    fun matToEValueGray(mat: Mat): EValue {
-      val pixelCount = mat.cols() * mat.rows()
-      val floatArray = FloatArray(pixelCount)
-
-      for (i in 0 until pixelCount) {
-        val row = i / mat.cols()
-        val col = i % mat.cols()
-        val pixel = mat.get(row, col)
-        floatArray[i] = pixel[0].toFloat()
-      }
-
-      return EValue.from(
-        Tensor.fromBlob(
-          floatArray,
-          longArrayOf(1, 1, mat.rows().toLong(), mat.cols().toLong()),
-        ),
-      )
-    }
-
-    fun eValueToMat(
-      array: FloatArray,
-      width: Int,
-      height: Int,
-    ): Mat {
-      val mat = Mat(height, width, CvType.CV_8UC3)
-
-      val pixelCount = width * height
-      for (i in 0 until pixelCount) {
-        val row = i / width
-        val col = i % width
-
-        val r = (array[i] * 255).toInt().toByte()
-        val g = (array[pixelCount + i] * 255).toInt().toByte()
-        val b = (array[2 * pixelCount + i] * 255).toInt().toByte()
-
-        val color = byteArrayOf(b, g, r)
-        mat.put(row, col, color)
-      }
-      return mat
-    }
-
-    fun saveToTempFile(
-      context: Context,
-      mat: Mat,
-    ): String {
-      try {
-        val uniqueID = UUID.randomUUID().toString()
-        val tempFile = File(context.cacheDir, "rn_executorch_$uniqueID.png")
-        Imgcodecs.imwrite(tempFile.absolutePath, mat)
-
-        return "file://${tempFile.absolutePath}"
-      } catch (e: Exception) {
-        throw Exception(ETError.FileWriteFailed.toString())
-      }
-    }
-
-    fun readImage(source: String): Mat {
-      val inputImage: Mat
-
-      val uri = Uri.parse(source)
-      val scheme = uri.scheme ?: ""
-
-      when {
-        scheme.equals("data", ignoreCase = true) -> {
-          // base64
-          val parts = source.split(",", limit = 2)
-          if (parts.size < 2) throw IllegalArgumentException(ETError.InvalidArgument.toString())
-
-          val encodedString = parts[1]
-          val data = Base64.decode(encodedString, Base64.DEFAULT)
-
-          val encodedData =
-            Mat(1, data.size, CvType.CV_8UC1).apply {
-              put(0, 0, data)
-            }
-          inputImage = Imgcodecs.imdecode(encodedData, Imgcodecs.IMREAD_COLOR)
-        }
-
-        scheme.equals("file", ignoreCase = true) -> {
-          // device storage
-          val path = uri.path
-          inputImage = Imgcodecs.imread(path, Imgcodecs.IMREAD_COLOR)
-        }
-
-        else -> {
-          // external source
-          val url = URL(source)
-          val connection = url.openConnection()
-          connection.connect()
-
-          val inputStream: InputStream = connection.getInputStream()
-          val data = inputStream.readBytes()
-          inputStream.close()
-
-          val encodedData =
-            Mat(1, data.size, CvType.CV_8UC1).apply {
-              put(0, 0, data)
-            }
-          inputImage = Imgcodecs.imdecode(encodedData, Imgcodecs.IMREAD_COLOR)
-        }
-      }
-
-      if (inputImage.empty()) {
-        throw IllegalArgumentException(ETError.InvalidArgument.toString())
-      }
-
-      return inputImage
-    }
-
-    fun resizeWithPadding(
-      img: Mat,
-      desiredWidth: Int,
-      desiredHeight: Int,
-    ): Mat {
-      val height = img.rows()
-      val width = img.cols()
-      val heightRatio = desiredHeight.toFloat() / height
-      val widthRatio = desiredWidth.toFloat() / width
-      val resizeRatio = minOf(heightRatio, widthRatio)
-      val newWidth = (width * resizeRatio).toInt()
-      val newHeight = (height * resizeRatio).toInt()
-
-      val resizedImg = Mat()
-      Imgproc.resize(
-        img,
-        resizedImg,
-        Size(newWidth.toDouble(), newHeight.toDouble()),
-        0.0,
-        0.0,
-        Imgproc.INTER_AREA,
-      )
-
-      val cornerPatchSize = maxOf(1, minOf(width, height) / 30)
-      val corners =
-        listOf(
-          img.submat(0, cornerPatchSize, 0, cornerPatchSize),
-          img.submat(0, cornerPatchSize, width - cornerPatchSize, width),
-          img.submat(height - cornerPatchSize, height, 0, cornerPatchSize),
-          img.submat(height - cornerPatchSize, height, width - cornerPatchSize, width),
-        )
-
-      var backgroundScalar = Core.mean(corners[0])
-      for (i in 1 until corners.size) {
-        val mean = Core.mean(corners[i])
-        backgroundScalar =
-          Scalar(
-            backgroundScalar.`val`[0] + mean.`val`[0],
-            backgroundScalar.`val`[1] + mean.`val`[1],
-            backgroundScalar.`val`[2] + mean.`val`[2],
-          )
-      }
-
-      backgroundScalar =
-        Scalar(
-          floor(backgroundScalar.`val`[0] / corners.size),
-          floor(backgroundScalar.`val`[1] / corners.size),
-          floor(backgroundScalar.`val`[2] / corners.size),
-        )
-
-      val deltaW = desiredWidth - newWidth
-      val deltaH = desiredHeight - newHeight
-      val top = deltaH / 2
-      val bottom = deltaH - top
-      val left = deltaW / 2
-      val right = deltaW - left
-
-      val centeredImg = Mat()
-      Core.copyMakeBorder(
-        resizedImg,
-        centeredImg,
-        top,
-        bottom,
-        left,
-        right,
-        Core.BORDER_CONSTANT,
-        backgroundScalar,
-      )
-
-      return centeredImg
-    }
-  }
-}

package/android/src/main/java/com/swmansion/rnexecutorch/utils/Numerical.kt

@@ -1,8 +0,0 @@
-package com.swmansion.rnexecutorch.utils
-
-fun softmax(x: Array<Float>): Array<Float> {
-  val max = x.maxOrNull()!!
-  val exps = x.map { kotlin.math.exp(it - max) }
-  val sum = exps.sum()
-  return exps.map { it / sum }.toTypedArray()
-}

package/android/src/main/java/com/swmansion/rnexecutorch/utils/ObjectDetectionUtils.kt

@@ -1,201 +0,0 @@
-package com.swmansion.rnexecutorch.utils
-
-import com.facebook.react.bridge.Arguments
-import com.facebook.react.bridge.WritableMap
-
-fun nms(
-  detections: MutableList<Detection>,
-  iouThreshold: Float,
-): List<Detection> {
-  if (detections.isEmpty()) {
-    return emptyList()
-  }
-
-  // Sort detections first by label, then by score (descending)
-  val sortedDetections = detections.sortedWith(compareBy({ it.label }, { -it.score }))
-
-  val result = mutableListOf<Detection>()
-
-  // Process NMS for each label group
-  var i = 0
-  while (i < sortedDetections.size) {
-    val currentLabel = sortedDetections[i].label
-
-    // Collect detections for the current label
-    val labelDetections = mutableListOf<Detection>()
-    while (i < sortedDetections.size && sortedDetections[i].label == currentLabel) {
-      labelDetections.add(sortedDetections[i])
-      i++
-    }
-
-    // Filter out detections with high IoU
-    val filteredLabelDetections = mutableListOf<Detection>()
-    while (labelDetections.isNotEmpty()) {
-      val current = labelDetections.removeAt(0)
-      filteredLabelDetections.add(current)
-
-      // Remove detections that overlap with the current detection above the IoU threshold
-      val iterator = labelDetections.iterator()
-      while (iterator.hasNext()) {
-        val other = iterator.next()
-        if (calculateIoU(current.bbox, other.bbox) > iouThreshold) {
-          iterator.remove() // Remove detection if IoU is above threshold
-        }
-      }
-    }
-
-    // Add the filtered detections to the result
-    result.addAll(filteredLabelDetections)
-  }
-
-  return result
-}
-
-fun calculateIoU(
-  bbox1: Bbox,
-  bbox2: Bbox,
-): Float {
-  val x1 = maxOf(bbox1.x1, bbox2.x1)
-  val y1 = maxOf(bbox1.y1, bbox2.y1)
-  val x2 = minOf(bbox1.x2, bbox2.x2)
-  val y2 = minOf(bbox1.y2, bbox2.y2)
-
-  val intersectionArea = maxOf(0f, x2 - x1) * maxOf(0f, y2 - y1)
-  val bbox1Area = (bbox1.x2 - bbox1.x1) * (bbox1.y2 - bbox1.y1)
-  val bbox2Area = (bbox2.x2 - bbox2.x1) * (bbox2.y2 - bbox2.y1)
-
-  val unionArea = bbox1Area + bbox2Area - intersectionArea
-  return if (unionArea == 0f) 0f else intersectionArea / unionArea
-}
-
-data class Bbox(
-  val x1: Float,
-  val y1: Float,
-  val x2: Float,
-  val y2: Float,
-) {
-  fun toWritableMap(): WritableMap {
-    val map = Arguments.createMap()
-    map.putDouble("x1", x1.toDouble())
-    map.putDouble("x2", x2.toDouble())
-    map.putDouble("y1", y1.toDouble())
-    map.putDouble("y2", y2.toDouble())
-    return map
-  }
-}
-
-data class Detection(
-  val bbox: Bbox,
-  val score: Float,
-  val label: CocoLabel,
-) {
-  fun toWritableMap(): WritableMap {
-    val map = Arguments.createMap()
-    map.putMap("bbox", bbox.toWritableMap())
-    map.putDouble("score", score.toDouble())
-    map.putString("label", label.name)
-    return map
-  }
-}
-
-enum class CocoLabel(
-  val id: Int,
-) {
-  PERSON(1),
-  BICYCLE(2),
-  CAR(3),
-  MOTORCYCLE(4),
-  AIRPLANE(5),
-  BUS(6),
-  TRAIN(7),
-  TRUCK(8),
-  BOAT(9),
-  TRAFFIC_LIGHT(10),
-  FIRE_HYDRANT(11),
-  STREET_SIGN(12),
-  STOP_SIGN(13),
-  PARKING(14),
-  BENCH(15),
-  BIRD(16),
-  CAT(17),
-  DOG(18),
-  HORSE(19),
-  SHEEP(20),
-  COW(21),
-  ELEPHANT(22),
-  BEAR(23),
-  ZEBRA(24),
-  GIRAFFE(25),
-  HAT(26),
-  BACKPACK(27),
-  UMBRELLA(28),
-  SHOE(29),
-  EYE(30),
-  HANDBAG(31),
-  TIE(32),
-  SUITCASE(33),
-  FRISBEE(34),
-  SKIS(35),
-  SNOWBOARD(36),
-  SPORTS(37),
-  KITE(38),
-  BASEBALL(39),
-  SKATEBOARD(41),
-  SURFBOARD(42),
-  TENNIS_RACKET(43),
-  BOTTLE(44),
-  PLATE(45),
-  WINE_GLASS(46),
-  CUP(47),
-  FORK(48),
-  KNIFE(49),
-  SPOON(50),
-  BOWL(51),
-  BANANA(52),
-  APPLE(53),
-  SANDWICH(54),
-  ORANGE(55),
-  BROCCOLI(56),
-  CARROT(57),
-  HOT_DOG(58),
-  PIZZA(59),
-  DONUT(60),
-  CAKE(61),
-  CHAIR(62),
-  COUCH(63),
-  POTTED_PLANT(64),
-  BED(65),
-  MIRROR(66),
-  DINING_TABLE(67),
-  WINDOW(68),
-  DESK(69),
-  TOILET(70),
-  DOOR(71),
-  TV(72),
-  LAPTOP(73),
-  MOUSE(74),
-  REMOTE(75),
-  KEYBOARD(76),
-  CELL_PHONE(77),
-  MICROWAVE(78),
-  OVEN(79),
-  TOASTER(80),
-  SINK(81),
-  REFRIGERATOR(82),
-  BLENDER(83),
-  BOOK(84),
-  CLOCK(85),
-  VASE(86),
-  SCISSORS(87),
-  TEDDY_BEAR(88),
-  HAIR_DRIER(89),
-  TOOTHBRUSH(90),
-  HAIR_BRUSH(91),
-  ;
-
-  companion object {
-    private val idToLabelMap = values().associateBy(CocoLabel::id)
-
-    fun fromId(id: Int): CocoLabel? = idToLabelMap[id]
-  }
-}

package/android/src/main/java/com/swmansion/rnexecutorch/utils/STFT.kt

@@ -1,50 +0,0 @@
-package com.swmansion.rnexecutorch.utils
-
-import org.jtransforms.fft.FloatFFT_1D
-import kotlin.math.PI
-import kotlin.math.cos
-import kotlin.math.sqrt
-
-class STFT public constructor(
-  var fftSize: Int = 512,
-  var hopLength: Int = 160,
-) {
-  private val fftModule = FloatFFT_1D(this.fftSize.toLong())
-  private val magnitudeScale = 1.0 / this.fftSize
-
-  // https://www.mathworks.com/help/signal/ref/hann.html
-  private val hannWindow = FloatArray(this.fftSize) { i -> 0.5f - 0.5f * cos(2f * PI.toFloat() * i / this.fftSize) }
-
-  fun fromWaveform(signal: FloatArray): FloatArray {
-    val numFftFrames = (signal.size - this.fftSize) / this.hopLength
-    // The output of FFT is always 2x smaller
-    val stft = FloatArray(numFftFrames * (this.fftSize / 2))
-
-    var windowStartIdx = 0
-    var outputIndex = 0
-    // TODO: i dont think the substraction at the end is needed
-    while (windowStartIdx + this.fftSize < signal.size - this.fftSize) {
-      val currentWindow = signal.copyOfRange(windowStartIdx, windowStartIdx + this.fftSize)
-      // Apply Hann window to the current slice
-      for (i in currentWindow.indices) currentWindow[i] *= this.hannWindow[i]
-
-      // Perform in-place FFT
-      this.fftModule.realForward(currentWindow)
-
-      stft[outputIndex++] = kotlin.math.abs(currentWindow[0])
-      for (i in 1 until this.fftSize / 2 - 1) {
-        val real = currentWindow[2 * i]
-        val imag = currentWindow[2 * i + 1]
-
-        val currentMagnitude = (sqrt(real * real + imag * imag) * this.magnitudeScale).toFloat()
-        // FIXME: we don't need that, but if we remove this we have to get rid of
-        // reversing this operation in the preprocessing part
-        stft[outputIndex++] = 20 * kotlin.math.log10(currentMagnitude)
-      }
-      // Nyquist frequency
-      stft[outputIndex++] = kotlin.math.abs(currentWindow[1])
-      windowStartIdx += this.hopLength
-    }
-    return stft
-  }
-}