react-native-executorch 0.4.8 → 0.5.1-rc.0

package/android/src/main/java/com/swmansion/rnexecutorch/StyleTransfer.kt

@@ -1,54 +0,0 @@
-package com.swmansion.rnexecutorch
-
-import android.util.Log
-import com.facebook.react.bridge.Promise
-import com.facebook.react.bridge.ReactApplicationContext
-import com.swmansion.rnexecutorch.models.styletransfer.StyleTransferModel
-import com.swmansion.rnexecutorch.utils.ETError
-import com.swmansion.rnexecutorch.utils.ImageProcessor
-import org.opencv.android.OpenCVLoader
-
-class StyleTransfer(
-  reactContext: ReactApplicationContext,
-) : NativeStyleTransferSpec(reactContext) {
-  private lateinit var styleTransferModel: StyleTransferModel
-
-  companion object {
-    const val NAME = "StyleTransfer"
-
-    init {
-      if (!OpenCVLoader.initLocal()) {
-        Log.d("rn_executorch", "OpenCV not loaded")
-      } else {
-        Log.d("rn_executorch", "OpenCV loaded")
-      }
-    }
-  }
-
-  override fun loadModule(
-    modelSource: String,
-    promise: Promise,
-  ) {
-    try {
-      styleTransferModel = StyleTransferModel(reactApplicationContext)
-      styleTransferModel.loadModel(modelSource)
-      promise.resolve(0)
-    } catch (e: Exception) {
-      promise.reject(e.message!!, ETError.InvalidModelSource.toString())
-    }
-  }
-
-  override fun forward(
-    input: String,
-    promise: Promise,
-  ) {
-    try {
-      val output = styleTransferModel.runModel(ImageProcessor.readImage(input))
-      promise.resolve(ImageProcessor.saveToTempFile(reactApplicationContext, output))
-    } catch (e: Exception) {
-      promise.reject(e.message!!, e.message)
-    }
-  }
-
-  override fun getName(): String = NAME
-}

package/android/src/main/java/com/swmansion/rnexecutorch/TextEmbeddings.kt

@@ -1,51 +0,0 @@
-package com.swmansion.rnexecutorch
-
-import com.facebook.react.bridge.Promise
-import com.facebook.react.bridge.ReactApplicationContext
-import com.facebook.react.bridge.WritableNativeArray
-import com.swmansion.rnexecutorch.models.textEmbeddings.TextEmbeddingsModel
-import com.swmansion.rnexecutorch.utils.ETError
-
-class TextEmbeddings(
-  reactContext: ReactApplicationContext,
-) : NativeTextEmbeddingsSpec(reactContext) {
-  private lateinit var textEmbeddingsModel: TextEmbeddingsModel
-
-  companion object {
-    const val NAME = "TextEmbeddings"
-  }
-
-  override fun loadModule(
-    modelSource: String,
-    tokenizerSource: String,
-    promise: Promise,
-  ) {
-    try {
-      textEmbeddingsModel = TextEmbeddingsModel(reactApplicationContext)
-
-      textEmbeddingsModel.loadModel(modelSource)
-      textEmbeddingsModel.loadTokenizer(tokenizerSource)
-
-      promise.resolve(0)
-    } catch (e: Exception) {
-      promise.reject(e.message!!, ETError.InvalidModelSource.toString())
-    }
-  }
-
-  override fun forward(
-    input: String,
-    promise: Promise,
-  ) {
-    try {
-      val output = textEmbeddingsModel.runModel(input)
-      val writableArray = WritableNativeArray()
-      output.forEach { writableArray.pushDouble(it) }
-
-      promise.resolve(writableArray)
-    } catch (e: Exception) {
-      promise.reject(e.message!!, e.message)
-    }
-  }
-
-  override fun getName(): String = NAME
-}

package/android/src/main/java/com/swmansion/rnexecutorch/Tokenizer.kt

@@ -1,86 +0,0 @@
-package com.swmansion.rnexecutorch
-
-import com.facebook.react.bridge.Promise
-import com.facebook.react.bridge.ReactApplicationContext
-import com.facebook.react.bridge.ReadableArray
-import com.swmansion.rnexecutorch.utils.ArrayUtils.Companion.createIntArray
-import com.swmansion.rnexecutorch.utils.ArrayUtils.Companion.createReadableArrayFromIntArray
-import com.swmansion.rnexecutorch.utils.ETError
-import org.pytorch.executorch.HuggingFaceTokenizer
-
-class Tokenizer(
-  reactContext: ReactApplicationContext,
-) : NativeTokenizerSpec(reactContext) {
-  private lateinit var tokenizer: HuggingFaceTokenizer
-
-  companion object {
-    const val NAME = "Tokenizer"
-  }
-
-  override fun loadModule(
-    tokenizerSource: String,
-    promise: Promise,
-  ) {
-    try {
-      tokenizer = HuggingFaceTokenizer(tokenizerSource)
-      promise.resolve(0)
-    } catch (e: Exception) {
-      promise.reject(e.message!!, ETError.InvalidModelSource.toString())
-    }
-  }
-
-  override fun decode(
-    input: ReadableArray,
-    skipSpecialTokens: Boolean,
-    promise: Promise,
-  ) {
-    try {
-      promise.resolve(tokenizer.decode(createIntArray(input), skipSpecialTokens))
-    } catch (e: Exception) {
-      promise.reject(e.message!!, ETError.UndefinedError.toString())
-    }
-  }
-
-  override fun encode(
-    input: String,
-    promise: Promise,
-  ) {
-    try {
-      promise.resolve(createReadableArrayFromIntArray(tokenizer.encode(input)))
-    } catch (e: Exception) {
-      promise.reject(e.message!!, ETError.UndefinedError.toString())
-    }
-  }
-
-  override fun getVocabSize(promise: Promise) {
-    try {
-      promise.resolve(tokenizer.vocabSize)
-    } catch (e: Exception) {
-      promise.reject(e.message!!, ETError.UndefinedError.toString())
-    }
-  }
-
-  override fun idToToken(
-    id: Double,
-    promise: Promise,
-  ) {
-    try {
-      promise.resolve(tokenizer.idToToken(id.toInt()))
-    } catch (e: Exception) {
-      promise.reject(e.message!!, ETError.UndefinedError.toString())
-    }
-  }
-
-  override fun tokenToId(
-    token: String,
-    promise: Promise,
-  ) {
-    try {
-      promise.resolve(tokenizer.tokenToId(token))
-    } catch (e: Exception) {
-      promise.reject(e.message!!, ETError.UndefinedError.toString())
-    }
-  }
-
-  override fun getName(): String = NAME
-}

package/android/src/main/java/com/swmansion/rnexecutorch/VerticalOCR.kt

@@ -1,179 +0,0 @@
-package com.swmansion.rnexecutorch
-
-import android.util.Log
-import com.facebook.react.bridge.Arguments
-import com.facebook.react.bridge.Promise
-import com.facebook.react.bridge.ReactApplicationContext
-import com.swmansion.rnexecutorch.models.ocr.Recognizer
-import com.swmansion.rnexecutorch.models.ocr.VerticalDetector
-import com.swmansion.rnexecutorch.models.ocr.utils.CTCLabelConverter
-import com.swmansion.rnexecutorch.models.ocr.utils.Constants
-import com.swmansion.rnexecutorch.models.ocr.utils.RecognizerUtils
-import com.swmansion.rnexecutorch.utils.ETError
-import com.swmansion.rnexecutorch.utils.ImageProcessor
-import org.opencv.android.OpenCVLoader
-import org.opencv.core.Core
-import org.opencv.core.Mat
-
-class VerticalOCR(
-  reactContext: ReactApplicationContext,
-) : NativeVerticalOCRSpec(reactContext) {
-  private lateinit var detectorLarge: VerticalDetector
-  private lateinit var detectorNarrow: VerticalDetector
-  private lateinit var recognizer: Recognizer
-  private lateinit var converter: CTCLabelConverter
-  private var independentCharacters = true
-
-  companion object {
-    const val NAME = "VerticalOCR"
-  }
-
-  init {
-    if (!OpenCVLoader.initLocal()) {
-      Log.d("rn_executorch", "OpenCV not loaded")
-    } else {
-      Log.d("rn_executorch", "OpenCV loaded")
-    }
-  }
-
-  override fun loadModule(
-    detectorLargeSource: String,
-    detectorNarrowSource: String,
-    recognizerSource: String,
-    symbols: String,
-    independentCharacters: Boolean,
-    promise: Promise,
-  ) {
-    try {
-      this.independentCharacters = independentCharacters
-      detectorLarge = VerticalDetector(false, reactApplicationContext)
-      detectorLarge.loadModel(detectorLargeSource)
-      detectorNarrow = VerticalDetector(true, reactApplicationContext)
-      detectorNarrow.loadModel(detectorNarrowSource)
-      recognizer = Recognizer(reactApplicationContext)
-      recognizer.loadModel(recognizerSource)
-
-      converter = CTCLabelConverter(symbols)
-
-      promise.resolve(0)
-    } catch (e: Exception) {
-      promise.reject(e.message!!, ETError.InvalidModelSource.toString())
-    }
-  }
-
-  override fun forward(
-    input: String,
-    promise: Promise,
-  ) {
-    try {
-      val inputImage = ImageProcessor.readImage(input)
-      val result = detectorLarge.runModel(inputImage)
-      val largeDetectorSize = detectorLarge.getModelImageSize()
-      val resizedImage =
-        ImageProcessor.resizeWithPadding(
-          inputImage,
-          largeDetectorSize.width.toInt(),
-          largeDetectorSize.height.toInt(),
-        )
-      val predictions = Arguments.createArray()
-      for (box in result) {
-        val cords = box.bBox
-        val boxWidth = cords[2].x - cords[0].x
-        val boxHeight = cords[2].y - cords[0].y
-
-        val boundingBox = RecognizerUtils.extractBoundingBox(cords)
-        val croppedImage = Mat(resizedImage, boundingBox)
-
-        val paddings =
-          RecognizerUtils.calculateResizeRatioAndPaddings(
-            inputImage.width(),
-            inputImage.height(),
-            largeDetectorSize.width.toInt(),
-            largeDetectorSize.height.toInt(),
-          )
-
-        var text = ""
-        var confidenceScore = 0.0
-        val boxResult = detectorNarrow.runModel(croppedImage)
-        val narrowDetectorSize = detectorNarrow.getModelImageSize()
-
-        val croppedCharacters = mutableListOf<Mat>()
-
-        for (characterBox in boxResult) {
-          val boxCords = characterBox.bBox
-          val paddingsBox =
-            RecognizerUtils.calculateResizeRatioAndPaddings(
-              boxWidth.toInt(),
-              boxHeight.toInt(),
-              narrowDetectorSize.width.toInt(),
-              narrowDetectorSize.height.toInt(),
-            )
-
-          var croppedCharacter =
-            RecognizerUtils.cropImageWithBoundingBox(
-              inputImage,
-              boxCords,
-              cords,
-              paddingsBox,
-              paddings,
-            )
-
-          if (this.independentCharacters) {
-            croppedCharacter = RecognizerUtils.cropSingleCharacter(croppedCharacter)
-            croppedCharacter = RecognizerUtils.normalizeForRecognizer(croppedCharacter, 0.0, true)
-            val recognitionResult = recognizer.runModel(croppedCharacter)
-            val predIndex = recognitionResult.first
-            val decodedText = converter.decodeGreedy(predIndex, predIndex.size)
-            text += decodedText[0]
-            confidenceScore += recognitionResult.second
-          } else {
-            croppedCharacters.add(croppedCharacter)
-          }
-        }
-
-        if (this.independentCharacters) {
-          confidenceScore /= boxResult.size
-        } else {
-          var mergedCharacters = Mat()
-          Core.hconcat(croppedCharacters, mergedCharacters)
-          mergedCharacters =
-            ImageProcessor.resizeWithPadding(
-              mergedCharacters,
-              Constants.LARGE_MODEL_WIDTH,
-              Constants.MODEL_HEIGHT,
-            )
-          mergedCharacters = RecognizerUtils.normalizeForRecognizer(mergedCharacters, 0.0)
-
-          val recognitionResult = recognizer.runModel(mergedCharacters)
-          val predIndex = recognitionResult.first
-          val decodedText = converter.decodeGreedy(predIndex, predIndex.size)
-
-          text = decodedText[0]
-          confidenceScore = recognitionResult.second
-        }
-
-        for (bBox in box.bBox) {
-          bBox.x =
-            (bBox.x - paddings["left"] as Int) * paddings["resizeRatio"] as Float
-          bBox.y =
-            (bBox.y - paddings["top"] as Int) * paddings["resizeRatio"] as Float
-        }
-
-        val resMap = Arguments.createMap()
-
-        resMap.putString("text", text)
-        resMap.putArray("bbox", box.toWritableArray())
-        resMap.putDouble("score", confidenceScore)
-
-        predictions.pushMap(resMap)
-      }
-
-      promise.resolve(predictions)
-    } catch (e: Exception) {
-      Log.d("rn_executorch", "Error running model: ${e.message}")
-      promise.reject(e.message!!, e.message)
-    }
-  }
-
-  override fun getName(): String = NAME
-}

package/android/src/main/java/com/swmansion/rnexecutorch/models/BaseModel.kt

@@ -1,54 +0,0 @@
-package com.swmansion.rnexecutorch.models
-
-import android.content.Context
-import com.swmansion.rnexecutorch.utils.ETError
-import org.pytorch.executorch.EValue
-import org.pytorch.executorch.Module
-import org.pytorch.executorch.Tensor
-
-abstract class BaseModel<Input, Output>(
-  val context: Context,
-) {
-  protected lateinit var module: Module
-
-  fun loadModel(modelSource: String) {
-    module = Module.load(modelSource)
-  }
-
-  protected fun forward(vararg inputs: EValue): Array<EValue> {
-    try {
-      val result = module.forward(*inputs)
-      return result
-    } catch (e: IllegalArgumentException) {
-      // The error is thrown when transformation to Tensor fails
-      throw Error(ETError.InvalidArgument.code.toString())
-    } catch (e: Exception) {
-      throw Error(e.message)
-    }
-  }
-
-  protected fun forward(
-    inputs: Array<FloatArray>,
-    shapes: Array<LongArray>,
-  ): Array<EValue> = this.execute("forward", inputs, shapes)
-
-  protected fun execute(
-    methodName: String,
-    inputs: Array<FloatArray>,
-    shapes: Array<LongArray>,
-  ): Array<EValue> {
-    // We want to convert each input to EValue, a data structure accepted by ExecuTorch's
-    // Module. The array below keeps track of that values.
-    try {
-      val executorchInputs = inputs.mapIndexed { index, _ -> EValue.from(Tensor.fromBlob(inputs[index], shapes[index])) }
-      val forwardResult = module.execute(methodName, *executorchInputs.toTypedArray())
-      return forwardResult
-    } catch (e: IllegalArgumentException) {
-      throw Error(ETError.InvalidArgument.code.toString())
-    } catch (e: Exception) {
-      throw Error(e.message)
-    }
-  }
-
-  abstract fun runModel(input: Input): Output
-}

package/android/src/main/java/com/swmansion/rnexecutorch/models/TextEmbeddings/TextEmbeddingsModel.kt

@@ -1,48 +0,0 @@
-package com.swmansion.rnexecutorch.models.textEmbeddings
-
-import com.facebook.react.bridge.ReactApplicationContext
-import com.swmansion.rnexecutorch.models.BaseModel
-import org.pytorch.executorch.EValue
-import org.pytorch.executorch.HuggingFaceTokenizer
-import org.pytorch.executorch.Tensor
-
-class TextEmbeddingsModel(
-  reactApplicationContext: ReactApplicationContext,
-) : BaseModel<String, DoubleArray>(reactApplicationContext) {
-  private lateinit var tokenizer: HuggingFaceTokenizer
-
-  fun loadTokenizer(tokenizerSource: String) {
-    tokenizer = HuggingFaceTokenizer(tokenizerSource)
-  }
-
-  fun preprocess(input: String): Array<LongArray> {
-    val inputIds = tokenizer.encode(input).map { it.toLong() }.toLongArray()
-    val attentionMask = inputIds.map { if (it != 0L) 1L else 0L }.toLongArray()
-    return arrayOf(inputIds, attentionMask) // Shape: [2, tokens]
-  }
-
-  fun postprocess(
-    modelOutput: FloatArray, // [tokens * embedding_dim]
-    attentionMask: LongArray, // [tokens]
-  ): DoubleArray {
-    val modelOutputDouble = modelOutput.map { it.toDouble() }.toDoubleArray()
-    val embeddings = TextEmbeddingsUtils.meanPooling(modelOutputDouble, attentionMask)
-    return TextEmbeddingsUtils.normalize(embeddings)
-  }
-
-  override fun runModel(input: String): DoubleArray {
-    val modelInput = preprocess(input)
-    val inputsIds = modelInput[0]
-    val attentionMask = modelInput[1]
-
-    val inputsIdsShape = longArrayOf(1, inputsIds.size.toLong())
-    val attentionMaskShape = longArrayOf(1, attentionMask.size.toLong())
-
-    val inputIdsEValue = EValue.from(Tensor.fromBlob(inputsIds, inputsIdsShape))
-    val attentionMaskEValue = EValue.from(Tensor.fromBlob(attentionMask, attentionMaskShape))
-
-    val modelOutput = forward(inputIdsEValue, attentionMaskEValue)[0].toTensor().dataAsFloatArray
-
-    return postprocess(modelOutput, attentionMask)
-  }
-}

package/android/src/main/java/com/swmansion/rnexecutorch/models/TextEmbeddings/TextEmbeddingsUtils.kt

@@ -1,37 +0,0 @@
-package com.swmansion.rnexecutorch.models.textEmbeddings
-
-import kotlin.math.sqrt
-
-class TextEmbeddingsUtils {
-  companion object {
-    fun meanPooling(
-      modelOutput: DoubleArray,
-      attentionMask: LongArray,
-    ): DoubleArray {
-      val attentionMaskLength = attentionMask.size
-      val modelOutputLength = modelOutput.size
-      val embeddingDim = modelOutputLength / attentionMaskLength
-
-      val result = DoubleArray(embeddingDim)
-      var sumMask = attentionMask.sum().toDouble()
-      sumMask = maxOf(sumMask, 1e-9)
-
-      for (i in 0 until embeddingDim) {
-        var sum = 0.0
-        for (j in 0 until attentionMaskLength) {
-          sum += modelOutput[j * embeddingDim + i] * attentionMask[j]
-        }
-        result[i] = sum / sumMask
-      }
-
-      return result
-    }
-
-    fun normalize(embeddings: DoubleArray): DoubleArray {
-      var sum = embeddings.sumOf { it * it }
-      sum = maxOf(sqrt(sum), 1e-9)
-
-      return embeddings.map { it / sum }.toDoubleArray()
-    }
-  }
-}

package/android/src/main/java/com/swmansion/rnexecutorch/models/classification/ClassificationModel.kt

@@ -1,46 +0,0 @@
-package com.swmansion.rnexecutorch.models.classification
-
-import com.facebook.react.bridge.ReactApplicationContext
-import com.swmansion.rnexecutorch.models.BaseModel
-import com.swmansion.rnexecutorch.utils.ImageProcessor
-import com.swmansion.rnexecutorch.utils.softmax
-import org.opencv.core.Mat
-import org.opencv.core.Size
-import org.opencv.imgproc.Imgproc
-import org.pytorch.executorch.EValue
-
-class ClassificationModel(
-  reactApplicationContext: ReactApplicationContext,
-) : BaseModel<Mat, Map<String, Float>>(reactApplicationContext) {
-  private fun getModelImageSize(): Size {
-    val inputShape = module.getInputShape(0)
-    val width = inputShape[inputShape.lastIndex]
-    val height = inputShape[inputShape.lastIndex - 1]
-
-    return Size(height.toDouble(), width.toDouble())
-  }
-
-  fun preprocess(input: Mat): EValue {
-    Imgproc.resize(input, input, getModelImageSize())
-    return ImageProcessor.matToEValue(input, module.getInputShape(0))
-  }
-
-  fun postprocess(output: Array<EValue>): Map<String, Float> {
-    val tensor = output[0].toTensor()
-    val probabilities = softmax(tensor.dataAsFloatArray.toTypedArray())
-
-    val result = mutableMapOf<String, Float>()
-
-    for (i in probabilities.indices) {
-      result[imagenet1k_v1_labels[i]] = probabilities[i]
-    }
-
-    return result
-  }
-
-  override fun runModel(input: Mat): Map<String, Float> {
-    val modelInput = preprocess(input)
-    val modelOutput = forward(modelInput)
-    return postprocess(modelOutput)
-  }
-}