react-native-executorch 0.3.3 → 0.4.1

package/README.md

@@ -8,13 +8,25 @@
 
 **ExecuTorch** is a novel framework created by Meta that enables running AI models on devices such as mobile phones or microcontrollers. React Native ExecuTorch bridges the gap between React Native and native platform capabilities, allowing developers to run AI models locally on mobile devices with state-of-the-art performance, without requiring deep knowledge of native code or machine learning internals.
 
+**Table of contents:**
+
+- [Compatibility](#compatibility)
+- [Ready-made models 🤖](#readymade-models-)
+- [Documentation 📚](#documentation-)
+- [🦙 Quickstart - Running Llama](#-quickstart---running-llama)
+- [Minimal supported versions](#minimal-supported-versions)
+- [Examples 📲](#examples-)
+- [Warning](#warning)
+- [License](#license)
+- [What's next?](#whats-next)
+
 ## Compatibility
 
 React Native Executorch supports only the [New React Native architecture](https://reactnative.dev/architecture/landing-page).
 
 If your app still runs on the old architecture, please consider upgrading to the New Architecture.
 
-## Readymade models 🤖
+## Ready-made models 🤖
 
 To run any AI model in ExecuTorch, you need to export it to a `.pte` format. If you're interested in experimenting with your own models, we highly encourage you to check out the [Python API](https://pypi.org/project/executorch/). If you prefer focusing on developing your React Native app, we will cover several common use cases. For more details, please refer to the documentation.
 
@@ -43,16 +55,17 @@ Add this to your component file:
 
 ```tsx
 import {
-  LLAMA3_2_3B_QLORA,
-  LLAMA3_2_3B_TOKENIZER,
   useLLM,
+  LLAMA3_2_1B,
+  LLAMA3_2_TOKENIZER_CONFIG,
 } from 'react-native-executorch';
 
 function MyComponent() {
   // Initialize the model 🚀
   const llama = useLLM({
-    modelSource: LLAMA3_2_3B_QLORA,
-    tokenizerSource: LLAMA3_2_3B_TOKENIZER,
+    modelSource: LLAMA3_2_1B,
+    tokenizerSource: LLAMA3_2_TOKENIZER,
+    tokenizerConfigSource: LLAMA3_2_TOKENIZER_CONFIG,
   });
   // ... rest of your component
 }
@@ -64,11 +77,14 @@ function MyComponent() {
 
 ```tsx
 const handleGenerate = async () => {
-  const prompt = 'The meaning of life is';
-
-  // Generate text based on your desired prompt
-  const response = await llama.generate(prompt);
-  console.log('Llama says:', response);
+  const chat = [
+    { role: 'system' content: 'You are a helpful assistant' }
+    { role: 'user', content: 'What is the meaning of life?' }
+  ];
+
+  // Chat completion
+  await llm.generate(chat);
+  console.log('Llama says:', llm.response);
 };
 ```
 
@@ -80,13 +96,14 @@ The minimal supported version is 17.0 for iOS and Android 13.
 
 https://github.com/user-attachments/assets/27ab3406-c7f1-4618-a981-6c86b53547ee
 
-We currently host two example apps demonstrating use cases of our library:
+We currently host a few example apps demonstrating use cases of our library:
 
+- examples/llm - chat application showcasing use of LLMs
 - examples/speech-to-text - Whisper and Moonshine models ready for transcription tasks
 - examples/computer-vision - computer vision related tasks
-- examples/llama - chat applications showcasing use of LLMs
+- examples/text-embeddings - computing text representations for semantic search
 
-If you would like to run it, navigate to it's project directory, for example `examples/llama` from the repository root and install dependencies with:
+If you would like to run it, navigate to it's project directory, for example `examples/llm` from the repository root and install dependencies with:
 
 ```bash
 yarn

package/android/build.gradle

@@ -103,7 +103,7 @@ dependencies {
   implementation "com.facebook.react:react-android:+"
   implementation 'org.opencv:opencv:4.10.0'
   implementation "org.jetbrains.kotlin:kotlin-stdlib:$kotlin_version"
-  implementation 'com.github.software-mansion:react-native-executorch:main-SNAPSHOT'
+  implementation(files("libs/executorch.aar"))
   implementation 'org.opencv:opencv:4.10.0'
   implementation("com.squareup.okhttp3:okhttp:4.9.2")
 }

package/android/src/main/java/com/swmansion/rnexecutorch/ETModule.kt

@@ -9,7 +9,6 @@ import com.swmansion.rnexecutorch.utils.ETError
 import com.swmansion.rnexecutorch.utils.TensorUtils
 import org.pytorch.executorch.EValue
 import org.pytorch.executorch.Module
-import java.net.URL
 
 class ETModule(
   reactContext: ReactApplicationContext,
@@ -23,7 +22,7 @@ class ETModule(
     modelSource: String,
     promise: Promise,
   ) {
-    module = Module.load(URL(modelSource).path)
+    module = Module.load(modelSource)
     promise.resolve(0)
   }
 

package/android/src/main/java/com/swmansion/rnexecutorch/ImageSegmentation.kt

@@ -0,0 +1,58 @@
+package com.swmansion.rnexecutorch
+
+import android.util.Log
+import com.facebook.react.bridge.Promise
+import com.facebook.react.bridge.ReactApplicationContext
+import com.facebook.react.bridge.ReadableArray
+import com.swmansion.rnexecutorch.models.imagesegmentation.ImageSegmentationModel
+import com.swmansion.rnexecutorch.utils.ETError
+import com.swmansion.rnexecutorch.utils.ImageProcessor
+import org.opencv.android.OpenCVLoader
+
+class ImageSegmentation(
+  reactContext: ReactApplicationContext,
+) : NativeImageSegmentationSpec(reactContext) {
+  private lateinit var model: ImageSegmentationModel
+
+  companion object {
+    const val NAME = "ImageSegmentation"
+
+    init {
+      if (!OpenCVLoader.initLocal()) {
+        Log.d("rn_executorch", "OpenCV not loaded")
+      } else {
+        Log.d("rn_executorch", "OpenCV loaded")
+      }
+    }
+  }
+
+  override fun loadModule(
+    modelSource: String,
+    promise: Promise,
+  ) {
+    try {
+      model = ImageSegmentationModel(reactApplicationContext)
+      model.loadModel(modelSource)
+      promise.resolve(0)
+    } catch (e: Exception) {
+      promise.reject(e.message!!, ETError.InvalidModelSource.toString())
+    }
+  }
+
+  override fun forward(
+    input: String,
+    classesOfInterest: ReadableArray,
+    resize: Boolean,
+    promise: Promise,
+  ) {
+    try {
+      val output =
+        model.runModel(Triple(ImageProcessor.readImage(input), classesOfInterest, resize))
+      promise.resolve(output)
+    } catch (e: Exception) {
+      promise.reject(e.message!!, e.message)
+    }
+  }
+
+  override fun getName(): String = NAME
+}

package/android/src/main/java/com/swmansion/rnexecutorch/LLM.kt

@@ -3,22 +3,13 @@ package com.swmansion.rnexecutorch
 import android.util.Log
 import com.facebook.react.bridge.Promise
 import com.facebook.react.bridge.ReactApplicationContext
-import com.facebook.react.bridge.ReadableArray
-import com.swmansion.rnexecutorch.utils.ArrayUtils
-import com.swmansion.rnexecutorch.utils.llms.ChatRole
-import com.swmansion.rnexecutorch.utils.llms.ConversationManager
-import com.swmansion.rnexecutorch.utils.llms.END_OF_TEXT_TOKEN
-import org.pytorch.executorch.LlamaCallback
-import org.pytorch.executorch.LlamaModule
-import java.net.URL
+import org.pytorch.executorch.extension.llm.LlmCallback
+import org.pytorch.executorch.extension.llm.LlmModule
 
 class LLM(
   reactContext: ReactApplicationContext,
-) : NativeLLMSpec(reactContext),
-  LlamaCallback {
-  private var llamaModule: LlamaModule? = null
-  private var tempLlamaResponse = StringBuilder()
-  private lateinit var conversationManager: ConversationManager
+) : NativeLLMSpec(reactContext), LlmCallback {
+  private var llmModule: LlmModule? = null
 
   override fun getName(): String = NAME
 
@@ -28,7 +19,6 @@ class LLM(
 
   override fun onResult(result: String) {
     emitOnToken(result)
-    this.tempLlamaResponse.append(result)
   }
 
   override fun onStats(tps: Float) {
@@ -38,59 +28,33 @@ class LLM(
   override fun loadLLM(
     modelSource: String,
     tokenizerSource: String,
-    systemPrompt: String,
-    messageHistory: ReadableArray,
-    contextWindowLength: Double,
     promise: Promise,
   ) {
     try {
-      this.conversationManager =
-        ConversationManager(
-          contextWindowLength.toInt(),
-          systemPrompt,
-          ArrayUtils.createMapArray<String>(messageHistory),
-        )
-      llamaModule = LlamaModule(1, URL(modelSource).path, URL(tokenizerSource).path, 0.7f)
-      this.tempLlamaResponse.clear()
+      llmModule = LlmModule(modelSource, tokenizerSource, 0.7f)
       promise.resolve("Model loaded successfully")
     } catch (e: Exception) {
       promise.reject("Model loading failed", e.message)
     }
   }
 
-  override fun runInference(
+  override fun forward(
     input: String,
     promise: Promise,
   ) {
-    this.conversationManager.addResponse(input, ChatRole.USER)
-    val conversation = this.conversationManager.getConversation()
-
     Thread {
-      llamaModule!!.generate(conversation, (conversation.length * 0.75).toInt() + 64, this, false)
-
-      // When we call .interrupt(), the LLM doesn't produce EOT token, that also could happen when the
-      // generated sequence length is larger than specified in the JNI callback, hence we check if EOT
-      // is there and if not, we append it to the output and emit the EOT token to the JS side.
-      if (!this.tempLlamaResponse.endsWith(END_OF_TEXT_TOKEN)) {
-        this.onResult(END_OF_TEXT_TOKEN)
-      }
-
-      // We want to add the LLM response to the conversation once all the tokens are generated.
-      // Each token is appended to the tempLlamaResponse StringBuilder in onResult callback.
-      this.conversationManager.addResponse(this.tempLlamaResponse.toString(), ChatRole.ASSISTANT)
-      this.tempLlamaResponse.clear()
-      Log.d("ExecutorchLib", this.conversationManager.getConversation())
-    }.start()
-
-    promise.resolve("Inference completed successfully")
+      llmModule!!.generate(input, this)
+      promise.resolve("Inference completed successfully")
+    }
+      .start()
   }
 
   override fun interrupt() {
-    llamaModule!!.stop()
+    llmModule!!.stop()
   }
 
-  override fun deleteModule() {
-    llamaModule = null
+  override fun releaseResources() {
+    llmModule = null
   }
 
   companion object {

package/android/src/main/java/com/swmansion/rnexecutorch/RnExecutorchPackage.kt

@@ -30,6 +30,12 @@ class RnExecutorchPackage : TurboReactPackage() {
       OCR(reactContext)
     } else if (name == VerticalOCR.NAME) {
       VerticalOCR(reactContext)
+    } else if (name == ImageSegmentation.NAME) {
+      ImageSegmentation(reactContext)
+    } else if (name == Tokenizer.NAME) {
+      Tokenizer(reactContext)
+    } else if (name == TextEmbeddings.NAME) {
+      TextEmbeddings(reactContext)
     } else {
       null
     }
@@ -115,6 +121,37 @@ class RnExecutorchPackage : TurboReactPackage() {
           false, // isCxxModule
           true,
         )
+
+      moduleInfos[ImageSegmentation.NAME] =
+        ReactModuleInfo(
+          ImageSegmentation.NAME,
+          ImageSegmentation.NAME,
+          false, // canOverrideExistingModule
+          false, // needsEagerInit
+          false, // isCxxModule
+          true,
+        )
+
+      moduleInfos[Tokenizer.NAME] =
+        ReactModuleInfo(
+          Tokenizer.NAME,
+          Tokenizer.NAME,
+          false, // canOverrideExistingModule
+          false, // needsEagerInit
+          false, // isCxxModule
+          true,
+        )
+
+      moduleInfos[TextEmbeddings.NAME] =
+        ReactModuleInfo(
+          TextEmbeddings.NAME,
+          TextEmbeddings.NAME,
+          false, // canOverrideExistingModule
+          false, // needsEagerInit
+          false, // isCxxModule
+          true,
+        )
+
       moduleInfos
     }
 }

package/android/src/main/java/com/swmansion/rnexecutorch/StyleTransfer.kt

@@ -3,7 +3,7 @@ package com.swmansion.rnexecutorch
 import android.util.Log
 import com.facebook.react.bridge.Promise
 import com.facebook.react.bridge.ReactApplicationContext
-import com.swmansion.rnexecutorch.models.StyleTransferModel
+import com.swmansion.rnexecutorch.models.styletransfer.StyleTransferModel
 import com.swmansion.rnexecutorch.utils.ETError
 import com.swmansion.rnexecutorch.utils.ImageProcessor
 import org.opencv.android.OpenCVLoader

package/android/src/main/java/com/swmansion/rnexecutorch/TextEmbeddings.kt

@@ -0,0 +1,51 @@
+package com.swmansion.rnexecutorch
+
+import com.facebook.react.bridge.Promise
+import com.facebook.react.bridge.ReactApplicationContext
+import com.facebook.react.bridge.WritableNativeArray
+import com.swmansion.rnexecutorch.models.textEmbeddings.TextEmbeddingsModel
+import com.swmansion.rnexecutorch.utils.ETError
+
+class TextEmbeddings(
+  reactContext: ReactApplicationContext,
+) : NativeTextEmbeddingsSpec(reactContext) {
+  private lateinit var textEmbeddingsModel: TextEmbeddingsModel
+
+  companion object {
+    const val NAME = "TextEmbeddings"
+  }
+
+  override fun loadModule(
+    modelSource: String,
+    tokenizerSource: String,
+    promise: Promise,
+  ) {
+    try {
+      textEmbeddingsModel = TextEmbeddingsModel(reactApplicationContext)
+
+      textEmbeddingsModel.loadModel(modelSource)
+      textEmbeddingsModel.loadTokenizer(tokenizerSource)
+
+      promise.resolve(0)
+    } catch (e: Exception) {
+      promise.reject(e.message!!, ETError.InvalidModelSource.toString())
+    }
+  }
+
+  override fun forward(
+    input: String,
+    promise: Promise,
+  ) {
+    try {
+      val output = textEmbeddingsModel.runModel(input)
+      val writableArray = WritableNativeArray()
+      output.forEach { writableArray.pushDouble(it) }
+
+      promise.resolve(writableArray)
+    } catch (e: Exception) {
+      promise.reject(e.message!!, e.message)
+    }
+  }
+
+  override fun getName(): String = NAME
+}

package/android/src/main/java/com/swmansion/rnexecutorch/Tokenizer.kt

@@ -0,0 +1,86 @@
+package com.swmansion.rnexecutorch
+
+import com.facebook.react.bridge.Promise
+import com.facebook.react.bridge.ReactApplicationContext
+import com.facebook.react.bridge.ReadableArray
+import com.swmansion.rnexecutorch.utils.ArrayUtils.Companion.createIntArray
+import com.swmansion.rnexecutorch.utils.ArrayUtils.Companion.createReadableArrayFromIntArray
+import com.swmansion.rnexecutorch.utils.ETError
+import org.pytorch.executorch.HuggingFaceTokenizer
+
+class Tokenizer(
+  reactContext: ReactApplicationContext,
+) : NativeTokenizerSpec(reactContext) {
+  private lateinit var tokenizer: HuggingFaceTokenizer
+
+  companion object {
+    const val NAME = "Tokenizer"
+  }
+
+  override fun loadModule(
+    tokenizerSource: String,
+    promise: Promise,
+  ) {
+    try {
+      tokenizer = HuggingFaceTokenizer(tokenizerSource)
+      promise.resolve(0)
+    } catch (e: Exception) {
+      promise.reject(e.message!!, ETError.InvalidModelSource.toString())
+    }
+  }
+
+  override fun decode(
+    input: ReadableArray,
+    skipSpecialTokens: Boolean,
+    promise: Promise,
+  ) {
+    try {
+      promise.resolve(tokenizer.decode(createIntArray(input), skipSpecialTokens))
+    } catch (e: Exception) {
+      promise.reject(e.message!!, ETError.UndefinedError.toString())
+    }
+  }
+
+  override fun encode(
+    input: String,
+    promise: Promise,
+  ) {
+    try {
+      promise.resolve(createReadableArrayFromIntArray(tokenizer.encode(input)))
+    } catch (e: Exception) {
+      promise.reject(e.message!!, ETError.UndefinedError.toString())
+    }
+  }
+
+  override fun getVocabSize(promise: Promise) {
+    try {
+      promise.resolve(tokenizer.vocabSize)
+    } catch (e: Exception) {
+      promise.reject(e.message!!, ETError.UndefinedError.toString())
+    }
+  }
+
+  override fun idToToken(
+    id: Double,
+    promise: Promise,
+  ) {
+    try {
+      promise.resolve(tokenizer.idToToken(id.toInt()))
+    } catch (e: Exception) {
+      promise.reject(e.message!!, ETError.UndefinedError.toString())
+    }
+  }
+
+  override fun tokenToId(
+    token: String,
+    promise: Promise,
+  ) {
+    try {
+      promise.resolve(tokenizer.tokenToId(token))
+    } catch (e: Exception) {
+      promise.reject(e.message!!, ETError.UndefinedError.toString())
+    }
+  }
+
+  override fun getName(): String = NAME
+}

package/android/src/main/java/com/swmansion/rnexecutorch/models/BaseModel.kt

@@ -5,7 +5,6 @@ import com.swmansion.rnexecutorch.utils.ETError
 import org.pytorch.executorch.EValue
 import org.pytorch.executorch.Module
 import org.pytorch.executorch.Tensor
-import java.net.URL
 
 abstract class BaseModel<Input, Output>(
   val context: Context,
@@ -13,12 +12,12 @@ abstract class BaseModel<Input, Output>(
   protected lateinit var module: Module
 
   fun loadModel(modelSource: String) {
-    module = Module.load(URL(modelSource).path)
+    module = Module.load(modelSource)
   }
 
-  protected fun forward(input: EValue): Array<EValue> {
+  protected fun forward(vararg inputs: EValue): Array<EValue> {
     try {
-      val result = module.forward(input)
+      val result = module.forward(*inputs)
       return result
     } catch (e: IllegalArgumentException) {
       // The error is thrown when transformation to Tensor fails

package/android/src/main/java/com/swmansion/rnexecutorch/models/TextEmbeddings/TextEmbeddingsModel.kt

@@ -0,0 +1,48 @@
+package com.swmansion.rnexecutorch.models.textEmbeddings
+
+import com.facebook.react.bridge.ReactApplicationContext
+import com.swmansion.rnexecutorch.models.BaseModel
+import org.pytorch.executorch.EValue
+import org.pytorch.executorch.HuggingFaceTokenizer
+import org.pytorch.executorch.Tensor
+
+class TextEmbeddingsModel(
+  reactApplicationContext: ReactApplicationContext,
+) : BaseModel<String, DoubleArray>(reactApplicationContext) {
+  private lateinit var tokenizer: HuggingFaceTokenizer
+
+  fun loadTokenizer(tokenizerSource: String) {
+    tokenizer = HuggingFaceTokenizer(tokenizerSource)
+  }
+
+  fun preprocess(input: String): Array<LongArray> {
+    val inputIds = tokenizer.encode(input).map { it.toLong() }.toLongArray()
+    val attentionMask = inputIds.map { if (it != 0L) 1L else 0L }.toLongArray()
+    return arrayOf(inputIds, attentionMask) // Shape: [2, tokens]
+  }
+
+  fun postprocess(
+    modelOutput: FloatArray, // [tokens * embedding_dim]
+    attentionMask: LongArray, // [tokens]
+  ): DoubleArray {
+    val modelOutputDouble = modelOutput.map { it.toDouble() }.toDoubleArray()
+    val embeddings = TextEmbeddingsUtils.meanPooling(modelOutputDouble, attentionMask)
+    return TextEmbeddingsUtils.normalize(embeddings)
+  }
+
+  override fun runModel(input: String): DoubleArray {
+    val modelInput = preprocess(input)
+    val inputsIds = modelInput[0]
+    val attentionMask = modelInput[1]
+
+    val inputsIdsShape = longArrayOf(1, inputsIds.size.toLong())
+    val attentionMaskShape = longArrayOf(1, attentionMask.size.toLong())
+
+    val inputIdsEValue = EValue.from(Tensor.fromBlob(inputsIds, inputsIdsShape))
+    val attentionMaskEValue = EValue.from(Tensor.fromBlob(attentionMask, attentionMaskShape))
+
+    val modelOutput = forward(inputIdsEValue, attentionMaskEValue)[0].toTensor().dataAsFloatArray
+
+    return postprocess(modelOutput, attentionMask)
+  }
+}

package/android/src/main/java/com/swmansion/rnexecutorch/models/TextEmbeddings/TextEmbeddingsUtils.kt

@@ -0,0 +1,37 @@
+package com.swmansion.rnexecutorch.models.textEmbeddings
+
+import kotlin.math.sqrt
+
+class TextEmbeddingsUtils {
+  companion object {
+    fun meanPooling(
+      modelOutput: DoubleArray,
+      attentionMask: LongArray,
+    ): DoubleArray {
+      val attentionMaskLength = attentionMask.size
+      val modelOutputLength = modelOutput.size
+      val embeddingDim = modelOutputLength / attentionMaskLength
+
+      val result = DoubleArray(embeddingDim)
+      var sumMask = attentionMask.sum().toDouble()
+      sumMask = maxOf(sumMask, 1e-9)
+
+      for (i in 0 until embeddingDim) {
+        var sum = 0.0
+        for (j in 0 until attentionMaskLength) {
+          sum += modelOutput[j * embeddingDim + i] * attentionMask[j]
+        }
+        result[i] = sum / sumMask
+      }
+
+      return result
+    }
+
+    fun normalize(embeddings: DoubleArray): DoubleArray {
+      var sum = embeddings.sumOf { it * it }
+      sum = maxOf(sqrt(sum), 1e-9)
+
+      return embeddings.map { it / sum }.toDoubleArray()
+    }
+  }
+}

package/android/src/main/java/com/swmansion/rnexecutorch/models/classification/ClassificationModel.kt

@@ -3,6 +3,7 @@ package com.swmansion.rnexecutorch.models.classification
 import com.facebook.react.bridge.ReactApplicationContext
 import com.swmansion.rnexecutorch.models.BaseModel
 import com.swmansion.rnexecutorch.utils.ImageProcessor
+import com.swmansion.rnexecutorch.utils.softmax
 import org.opencv.core.Mat
 import org.opencv.core.Size
 import org.opencv.imgproc.Imgproc

package/android/src/main/java/com/swmansion/rnexecutorch/models/imageSegmentation/Constants.kt

@@ -0,0 +1,26 @@
+package com.swmansion.rnexecutorch.models.imagesegmentation
+
+val deeplabv3_resnet50_labels: Array<String> =
+  arrayOf(
+    "BACKGROUND",
+    "AEROPLANE",
+    "BICYCLE",
+    "BIRD",
+    "BOAT",
+    "BOTTLE",
+    "BUS",
+    "CAR",
+    "CAT",
+    "CHAIR",
+    "COW",
+    "DININGTABLE",
+    "DOG",
+    "HORSE",
+    "MOTORBIKE",
+    "PERSON",
+    "POTTEDPLANT",
+    "SHEEP",
+    "SOFA",
+    "TRAIN",
+    "TVMONITOR",
+  )