react-native-executorch 0.4.9 → 0.5.0-nightly-6f4dd53-20251211

package/LICENSE

@@ -1,79 +0,0 @@
-MIT License
-
-Copyright (c) 2024 Software Mansion
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-This software includes components from the ExecuTorch library, which is licensed under the BSD 3-Clause license. The license and copyright notice for ExecuTorch are as follows:
-
-Copyright (c) Meta Platforms, Inc. and affiliates.
-Copyright 2023 Arm Limited and/or its affiliates.
-Copyright (c) Qualcomm Innovation Center, Inc.
-Copyright (c) 2023 Apple Inc.
-Copyright (c) 2024 MediaTek Inc.
-
-Redistribution and use in source and binary forms, with or without modification,
-are permitted provided that the following conditions are met:
-
-- Redistributions of source code must retain the above copyright notice, this
-  list of conditions and the following disclaimer.
-
-- Redistributions in binary form must reproduce the above copyright notice,
-  this list of conditions and the following disclaimer in the documentation
-  and/or other materials provided with the distribution.
-
-- Neither the name Meta nor the names of its contributors may be used to
-  endorse or promote products derived from this software without specific
-  prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
-FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
-
-This software includes components from the JTransforms library. The license and copyright notice for this library are as follows:
-JTransforms
-Copyright (c) 2007 onward, Piotr Wendykier
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
-1. Redistributions of source code must retain the above copyright notice, this
-   list of conditions and the following disclaimer. 
-2. Redistributions in binary form must reproduce the above copyright notice,
-   this list of conditions and the following disclaimer in the documentation
-   and/or other materials provided with the distribution.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
-ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
-ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
-ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

package/android/src/main/java/com/swmansion/rnexecutorch/Classification.kt

@@ -1,64 +0,0 @@
-package com.swmansion.rnexecutorch
-
-import android.util.Log
-import com.facebook.react.bridge.Arguments
-import com.facebook.react.bridge.Promise
-import com.facebook.react.bridge.ReactApplicationContext
-import com.facebook.react.bridge.WritableMap
-import com.swmansion.rnexecutorch.models.classification.ClassificationModel
-import com.swmansion.rnexecutorch.utils.ETError
-import com.swmansion.rnexecutorch.utils.ImageProcessor
-import org.opencv.android.OpenCVLoader
-
-class Classification(
-  reactContext: ReactApplicationContext,
-) : NativeClassificationSpec(reactContext) {
-  private lateinit var classificationModel: ClassificationModel
-
-  companion object {
-    const val NAME = "Classification"
-
-    init {
-      if (!OpenCVLoader.initLocal()) {
-        Log.d("rn_executorch", "OpenCV not loaded")
-      } else {
-        Log.d("rn_executorch", "OpenCV loaded")
-      }
-    }
-  }
-
-  override fun loadModule(
-    modelSource: String,
-    promise: Promise,
-  ) {
-    try {
-      classificationModel = ClassificationModel(reactApplicationContext)
-      classificationModel.loadModel(modelSource)
-      promise.resolve(0)
-    } catch (e: Exception) {
-      promise.reject(e.message!!, ETError.InvalidModelSource.toString())
-    }
-  }
-
-  override fun forward(
-    input: String,
-    promise: Promise,
-  ) {
-    try {
-      val image = ImageProcessor.readImage(input)
-      val output = classificationModel.runModel(image)
-
-      val writableMap: WritableMap = Arguments.createMap()
-
-      for ((key, value) in output) {
-        writableMap.putDouble(key, value.toDouble())
-      }
-
-      promise.resolve(writableMap)
-    } catch (e: Exception) {
-      promise.reject(e.message!!, e.message)
-    }
-  }
-
-  override fun getName(): String = NAME
-}

package/android/src/main/java/com/swmansion/rnexecutorch/ETModule.kt

@@ -1,90 +0,0 @@
-package com.swmansion.rnexecutorch
-
-import com.facebook.react.bridge.Arguments
-import com.facebook.react.bridge.Promise
-import com.facebook.react.bridge.ReactApplicationContext
-import com.facebook.react.bridge.ReadableArray
-import com.swmansion.rnexecutorch.utils.ArrayUtils
-import com.swmansion.rnexecutorch.utils.ETError
-import com.swmansion.rnexecutorch.utils.TensorUtils
-import org.pytorch.executorch.EValue
-import org.pytorch.executorch.Module
-
-class ETModule(
-  reactContext: ReactApplicationContext,
-) : NativeETModuleSpec(reactContext) {
-  private lateinit var module: Module
-  private var reactApplicationContext = reactContext
-
-  override fun getName(): String = NAME
-
-  override fun loadModule(
-    modelSource: String,
-    promise: Promise,
-  ) {
-    module = Module.load(modelSource)
-    promise.resolve(0)
-  }
-
-  override fun loadMethod(
-    methodName: String,
-    promise: Promise,
-  ) {
-    val result = module.loadMethod(methodName)
-    if (result != 0) {
-      promise.reject("Method loading failed", result.toString())
-      return
-    }
-
-    promise.resolve(result)
-  }
-
-  override fun forward(
-    inputs: ReadableArray,
-    shapes: ReadableArray,
-    inputTypes: ReadableArray,
-    promise: Promise,
-  ) {
-    val inputEValues = ArrayList<EValue>()
-    try {
-      for (i in 0 until inputs.size()) {
-        val currentInput =
-          inputs.getArray(i)
-            ?: throw Exception(ETError.InvalidArgument.code.toString())
-        val currentShape =
-          shapes.getArray(i)
-            ?: throw Exception(ETError.InvalidArgument.code.toString())
-        val currentInputType = inputTypes.getInt(i)
-
-        val currentEValue =
-          TensorUtils.getExecutorchInput(
-            currentInput,
-            ArrayUtils.createLongArray(currentShape),
-            currentInputType,
-          )
-
-        inputEValues.add(currentEValue)
-      }
-
-      val forwardOutputs = module.forward(*inputEValues.toTypedArray())
-      val outputArray = Arguments.createArray()
-
-      for (output in forwardOutputs) {
-        val arr = ArrayUtils.createReadableArrayFromTensor(output.toTensor())
-        outputArray.pushArray(arr)
-      }
-      promise.resolve(outputArray)
-    } catch (e: IllegalArgumentException) {
-      // The error is thrown when transformation to Tensor fails
-      promise.reject("Forward Failed Execution", ETError.InvalidArgument.code.toString())
-      return
-    } catch (e: Exception) {
-      promise.reject("Forward Failed Execution", e.message!!)
-      return
-    }
-  }
-
-  companion object {
-    const val NAME = "ETModule"
-  }
-}

package/android/src/main/java/com/swmansion/rnexecutorch/ImageSegmentation.kt

@@ -1,58 +0,0 @@
-package com.swmansion.rnexecutorch
-
-import android.util.Log
-import com.facebook.react.bridge.Promise
-import com.facebook.react.bridge.ReactApplicationContext
-import com.facebook.react.bridge.ReadableArray
-import com.swmansion.rnexecutorch.models.imagesegmentation.ImageSegmentationModel
-import com.swmansion.rnexecutorch.utils.ETError
-import com.swmansion.rnexecutorch.utils.ImageProcessor
-import org.opencv.android.OpenCVLoader
-
-class ImageSegmentation(
-  reactContext: ReactApplicationContext,
-) : NativeImageSegmentationSpec(reactContext) {
-  private lateinit var model: ImageSegmentationModel
-
-  companion object {
-    const val NAME = "ImageSegmentation"
-
-    init {
-      if (!OpenCVLoader.initLocal()) {
-        Log.d("rn_executorch", "OpenCV not loaded")
-      } else {
-        Log.d("rn_executorch", "OpenCV loaded")
-      }
-    }
-  }
-
-  override fun loadModule(
-    modelSource: String,
-    promise: Promise,
-  ) {
-    try {
-      model = ImageSegmentationModel(reactApplicationContext)
-      model.loadModel(modelSource)
-      promise.resolve(0)
-    } catch (e: Exception) {
-      promise.reject(e.message!!, ETError.InvalidModelSource.toString())
-    }
-  }
-
-  override fun forward(
-    input: String,
-    classesOfInterest: ReadableArray,
-    resize: Boolean,
-    promise: Promise,
-  ) {
-    try {
-      val output =
-        model.runModel(Triple(ImageProcessor.readImage(input), classesOfInterest, resize))
-      promise.resolve(output)
-    } catch (e: Exception) {
-      promise.reject(e.message!!, e.message)
-    }
-  }
-
-  override fun getName(): String = NAME
-}

package/android/src/main/java/com/swmansion/rnexecutorch/LLM.kt

@@ -1,63 +0,0 @@
-package com.swmansion.rnexecutorch
-
-import android.util.Log
-import com.facebook.react.bridge.Promise
-import com.facebook.react.bridge.ReactApplicationContext
-import org.pytorch.executorch.extension.llm.LlmCallback
-import org.pytorch.executorch.extension.llm.LlmModule
-
-class LLM(
-  reactContext: ReactApplicationContext,
-) : NativeLLMSpec(reactContext), LlmCallback {
-  private var llmModule: LlmModule? = null
-
-  override fun getName(): String = NAME
-
-  override fun initialize() {
-    super.initialize()
-  }
-
-  override fun onResult(result: String) {
-    emitOnToken(result)
-  }
-
-  override fun onStats(tps: Float) {
-    Log.d("rn_executorch", "TPS: $tps")
-  }
-
-  override fun loadLLM(
-    modelSource: String,
-    tokenizerSource: String,
-    promise: Promise,
-  ) {
-    try {
-      llmModule = LlmModule(modelSource, tokenizerSource, 0.7f)
-      promise.resolve("Model loaded successfully")
-    } catch (e: Exception) {
-      promise.reject("Model loading failed", e.message)
-    }
-  }
-
-  override fun forward(
-    input: String,
-    promise: Promise,
-  ) {
-    Thread {
-      llmModule!!.generate(input, this)
-      promise.resolve("Inference completed successfully")
-    }
-      .start()
-  }
-
-  override fun interrupt() {
-    llmModule!!.stop()
-  }
-
-  override fun releaseResources() {
-    llmModule = null
-  }
-
-  companion object {
-    const val NAME = "LLM"
-  }
-}

package/android/src/main/java/com/swmansion/rnexecutorch/OCR.kt

@@ -1,90 +0,0 @@
-package com.swmansion.rnexecutorch
-
-import android.util.Log
-import com.facebook.react.bridge.Promise
-import com.facebook.react.bridge.ReactApplicationContext
-import com.swmansion.rnexecutorch.models.ocr.Detector
-import com.swmansion.rnexecutorch.models.ocr.RecognitionHandler
-import com.swmansion.rnexecutorch.models.ocr.utils.Constants
-import com.swmansion.rnexecutorch.utils.ETError
-import com.swmansion.rnexecutorch.utils.ImageProcessor
-import org.opencv.android.OpenCVLoader
-import org.opencv.imgproc.Imgproc
-
-class OCR(
-  reactContext: ReactApplicationContext,
-) : NativeOCRSpec(reactContext) {
-  private lateinit var detector: Detector
-  private lateinit var recognitionHandler: RecognitionHandler
-
-  companion object {
-    const val NAME = "OCR"
-  }
-
-  init {
-    if (!OpenCVLoader.initLocal()) {
-      Log.d("rn_executorch", "OpenCV not loaded")
-    } else {
-      Log.d("rn_executorch", "OpenCV loaded")
-    }
-  }
-
-  override fun loadModule(
-    detectorSource: String,
-    recognizerSourceLarge: String,
-    recognizerSourceMedium: String,
-    recognizerSourceSmall: String,
-    symbols: String,
-    promise: Promise,
-  ) {
-    try {
-      detector = Detector(reactApplicationContext)
-      detector.loadModel(detectorSource)
-
-      recognitionHandler =
-        RecognitionHandler(
-          symbols,
-          reactApplicationContext,
-        )
-
-      recognitionHandler.loadRecognizers(
-        recognizerSourceLarge,
-        recognizerSourceMedium,
-        recognizerSourceSmall,
-      ) { _, errorRecognizer ->
-        if (errorRecognizer != null) {
-          throw Error(errorRecognizer.message!!)
-        }
-
-        promise.resolve(0)
-      }
-    } catch (e: Exception) {
-      promise.reject(e.message!!, ETError.InvalidModelSource.toString())
-    }
-  }
-
-  override fun forward(
-    input: String,
-    promise: Promise,
-  ) {
-    try {
-      val inputImage = ImageProcessor.readImage(input)
-      val bBoxesList = detector.runModel(inputImage)
-      val detectorSize = detector.getModelImageSize()
-      Imgproc.cvtColor(inputImage, inputImage, Imgproc.COLOR_BGR2GRAY)
-      val result =
-        recognitionHandler.recognize(
-          bBoxesList,
-          inputImage,
-          (detectorSize.width * Constants.RECOGNIZER_RATIO).toInt(),
-          (detectorSize.height * Constants.RECOGNIZER_RATIO).toInt(),
-        )
-      promise.resolve(result)
-    } catch (e: Exception) {
-      Log.d("rn_executorch", "Error running model: ${e.message}")
-      promise.reject(e.message!!, e.message)
-    }
-  }
-
-  override fun getName(): String = NAME
-}

package/android/src/main/java/com/swmansion/rnexecutorch/ObjectDetection.kt

@@ -1,64 +0,0 @@
-package com.swmansion.rnexecutorch
-
-import android.util.Log
-import com.facebook.react.bridge.Arguments
-import com.facebook.react.bridge.Promise
-import com.facebook.react.bridge.ReactApplicationContext
-import com.facebook.react.bridge.WritableArray
-import com.swmansion.rnexecutorch.models.objectdetection.SSDLiteLargeModel
-import com.swmansion.rnexecutorch.utils.ETError
-import com.swmansion.rnexecutorch.utils.ImageProcessor
-import org.opencv.android.OpenCVLoader
-
-class ObjectDetection(
-  reactContext: ReactApplicationContext,
-) : NativeObjectDetectionSpec(reactContext) {
-  private lateinit var ssdLiteLarge: SSDLiteLargeModel
-
-  companion object {
-    const val NAME = "ObjectDetection"
-  }
-
-  init {
-    if (!OpenCVLoader.initLocal()) {
-      Log.d("rn_executorch", "OpenCV not loaded")
-    } else {
-      Log.d("rn_executorch", "OpenCV loaded")
-    }
-  }
-
-  override fun loadModule(
-    modelSource: String,
-    promise: Promise,
-  ) {
-    try {
-      ssdLiteLarge = SSDLiteLargeModel(reactApplicationContext)
-      ssdLiteLarge.loadModel(modelSource)
-      promise.resolve(0)
-    } catch (e: Exception) {
-      promise.reject(e.message!!, ETError.InvalidModelSource.toString())
-    }
-  }
-
-  override fun forward(
-    input: String,
-    promise: Promise,
-  ) {
-    try {
-      val inputImage = ImageProcessor.readImage(input)
-      val output = ssdLiteLarge.runModel(inputImage)
-      val outputWritableArray: WritableArray = Arguments.createArray()
-      output
-        .map { detection ->
-          detection.toWritableMap()
-        }.forEach { writableMap ->
-          outputWritableArray.pushMap(writableMap)
-        }
-      promise.resolve(outputWritableArray)
-    } catch (e: Exception) {
-      promise.reject(e.message!!, e.message)
-    }
-  }
-
-  override fun getName(): String = NAME
-}

package/android/src/main/java/com/swmansion/rnexecutorch/SpeechToText.kt

@@ -1,91 +0,0 @@
-package com.swmansion.rnexecutorch
-
-import com.facebook.react.bridge.Promise
-import com.facebook.react.bridge.ReactApplicationContext
-import com.facebook.react.bridge.ReadableArray
-import com.swmansion.rnexecutorch.models.speechtotext.BaseS2TModule
-import com.swmansion.rnexecutorch.models.speechtotext.Moonshine
-import com.swmansion.rnexecutorch.models.speechtotext.MoonshineDecoder
-import com.swmansion.rnexecutorch.models.speechtotext.MoonshineEncoder
-import com.swmansion.rnexecutorch.models.speechtotext.Whisper
-import com.swmansion.rnexecutorch.models.speechtotext.WhisperDecoder
-import com.swmansion.rnexecutorch.models.speechtotext.WhisperEncoder
-import com.swmansion.rnexecutorch.utils.ArrayUtils
-import com.swmansion.rnexecutorch.utils.ArrayUtils.Companion.writableArrayToEValue
-import com.swmansion.rnexecutorch.utils.ETError
-
-class SpeechToText(
-  reactContext: ReactApplicationContext,
-) : NativeSpeechToTextSpec(reactContext) {
-  private lateinit var speechToTextModule: BaseS2TModule
-
-  companion object {
-    const val NAME = "SpeechToText"
-  }
-
-  override fun loadModule(
-    modelName: String,
-    modelSources: ReadableArray,
-    promise: Promise,
-  ) {
-    try {
-      if (modelName == "moonshine") {
-        this.speechToTextModule = Moonshine()
-        this.speechToTextModule.encoder = MoonshineEncoder(reactApplicationContext)
-        this.speechToTextModule.decoder = MoonshineDecoder(reactApplicationContext)
-      }
-      if (modelName == "whisper") {
-        this.speechToTextModule = Whisper()
-        this.speechToTextModule.encoder = WhisperEncoder(reactApplicationContext)
-        this.speechToTextModule.decoder = WhisperDecoder(reactApplicationContext)
-      }
-    } catch (e: Exception) {
-      promise.reject(e.message!!, ETError.InvalidModelSource.toString())
-      return
-    }
-
-    try {
-      this.speechToTextModule.loadModel(modelSources.getString(0)!!, modelSources.getString(1)!!)
-      promise.resolve(0)
-    } catch (e: Exception) {
-      promise.reject(e.message!!, ETError.InvalidModelSource.toString())
-    }
-  }
-
-  override fun generate(
-    waveform: ReadableArray,
-    promise: Promise,
-  ) {
-    val encoding = writableArrayToEValue(this.speechToTextModule.encode(waveform))
-    val generatedTokens = mutableListOf(this.speechToTextModule.startToken)
-    var lastToken = 0
-    Thread {
-      while (lastToken != this.speechToTextModule.eosToken) {
-        // TODO uncomment, for now
-        //        lastToken = this.speechToTextModule.decode(generatedTokens, encoding)
-        emitOnToken(lastToken.toDouble())
-        generatedTokens.add(lastToken)
-      }
-      val generatedTokensReadableArray =
-        ArrayUtils.createReadableArrayFromIntArray(generatedTokens.toIntArray())
-      promise.resolve(generatedTokensReadableArray)
-    }.start()
-  }
-
-  override fun encode(
-    waveform: ReadableArray,
-    promise: Promise,
-  ) {
-    promise.resolve(this.speechToTextModule.encode(waveform))
-  }
-
-  override fun decode(
-    prevTokens: ReadableArray,
-    encoderOutput: ReadableArray,
-    promise: Promise,
-  ) {
-    promise.resolve(this.speechToTextModule.decode(prevTokens, encoderOutput))
-  }
-
-  override fun getName(): String = NAME
-}

package/android/src/main/java/com/swmansion/rnexecutorch/StyleTransfer.kt

@@ -1,54 +0,0 @@
-package com.swmansion.rnexecutorch
-
-import android.util.Log
-import com.facebook.react.bridge.Promise
-import com.facebook.react.bridge.ReactApplicationContext
-import com.swmansion.rnexecutorch.models.styletransfer.StyleTransferModel
-import com.swmansion.rnexecutorch.utils.ETError
-import com.swmansion.rnexecutorch.utils.ImageProcessor
-import org.opencv.android.OpenCVLoader
-
-class StyleTransfer(
-  reactContext: ReactApplicationContext,
-) : NativeStyleTransferSpec(reactContext) {
-  private lateinit var styleTransferModel: StyleTransferModel
-
-  companion object {
-    const val NAME = "StyleTransfer"
-
-    init {
-      if (!OpenCVLoader.initLocal()) {
-        Log.d("rn_executorch", "OpenCV not loaded")
-      } else {
-        Log.d("rn_executorch", "OpenCV loaded")
-      }
-    }
-  }
-
-  override fun loadModule(
-    modelSource: String,
-    promise: Promise,
-  ) {
-    try {
-      styleTransferModel = StyleTransferModel(reactApplicationContext)
-      styleTransferModel.loadModel(modelSource)
-      promise.resolve(0)
-    } catch (e: Exception) {
-      promise.reject(e.message!!, ETError.InvalidModelSource.toString())
-    }
-  }
-
-  override fun forward(
-    input: String,
-    promise: Promise,
-  ) {
-    try {
-      val output = styleTransferModel.runModel(ImageProcessor.readImage(input))
-      promise.resolve(ImageProcessor.saveToTempFile(reactApplicationContext, output))
-    } catch (e: Exception) {
-      promise.reject(e.message!!, e.message)
-    }
-  }
-
-  override fun getName(): String = NAME
-}