react-native-executorch 0.5.1-rc.0 → 0.5.2

package/README.md

@@ -0,0 +1,132 @@
+<div align="right">
+  <h1 align="left" style="display:inline-block">React Native ExecuTorch 
+    <!-- Discord Badge -->
+    <a href="https://discord.gg/ZGqqY55qkP">
+      <img src="https://img.shields.io/badge/Discord-Join%20Us-00008B?logo=discord&logoColor=white&style=for-the-badge" alt="Join our Discord community">
+    </a>
+  </h1>
+</div>
+
+![Software Mansion banner](https://github.com/user-attachments/assets/fa2c4735-e75c-4cc1-970d-88905d95e3a4)
+
+**React Native ExecuTorch** provides a declarative way to run AI models on-device using React Native, powered by **ExecuTorch** :rocket:. It offers out-of-the-box support for a wide range of LLMs, computer vision models, and more. Visit our [HuggingFace](https://huggingface.co/software-mansion) page to explore these models.
+
+**ExecuTorch**, developed by Meta, is a novel framework allowing AI model execution on devices like mobile phones or microcontrollers.
+
+React Native ExecuTorch bridges the gap between React Native and native platform capabilities, enabling developers to efficiently run local AI models on mobile devices. This can be achieved without the need for extensive expertise in native programming or machine learning.
+
+[![npm version](https://img.shields.io/npm/v/react-native-executorch?color=00008B)](https://www.npmjs.com/package/react-native-executorch)
+[![CI](https://github.com/software-mansion/react-native-executorch/actions/workflows/ci.yml/badge.svg)](https://github.com/software-mansion/react-native-executorch/actions/workflows/ci.yml)
+
+**Table of contents:**
+
+- [:yin_yang: Supported versions](#yin_yang-supported-versions)
+- [:robot: Ready-made models](#robot-ready-made-models)
+- [:books: Documentation](#books-documentation)
+- [:llama: Quickstart - Running Llama](#llama-quickstart---running-llama)
+- [:calling: Examples](#calling-examples)
+- [:balance_scale: License](#balance_scale-license)
+- [:soon: What's next?](#soon-whats-next)
+
+## :yin_yang: Supported versions
+
+The minimal supported version are: 
+* iOS 17.0
+* Android 13
+* React Native 0.76
+
+> [!IMPORTANT]  
+> React Native Executorch supports only the [New React Native architecture](https://reactnative.dev/architecture/landing-page).
+
+## :robot: Ready-made models
+
+Our library has a number of ready-to-use AI models; a complete list is available in the documentation. If you're interested in running your own AI model, you need to first export it to the `.pte` format. Instructions on how to do this are available in the [Python API](https://pypi.org/project/executorch/).
+
+## :books: Documentation
+
+Check out how our library can help you build your React Native AI features by visiting our docs:  
+https://docs.swmansion.com/react-native-executorch
+
+## :llama: **Quickstart - Running Llama**
+
+**Get started with AI-powered text generation in 3 easy steps!**
+
+### :one: **Installation**
+
+```bash
+# Install the package
+yarn add react-native-executorch
+# Depending on the platform, choose either iOS or Android
+yarn expo run:< ios | android >
+```
+
+### :two: **Setup & Initialization**
+
+Add this to your component file:
+
+```tsx
+import {
+  useLLM,
+  LLAMA3_2_1B,
+  Message
+} from 'react-native-executorch';
+
+function MyComponent() {
+  // Initialize the model 🚀
+  const llm = useLLM({ model: LLAMA3_2_1B });
+  // ... rest of your component
+}
+```
+
+### :three: **Run the model!**
+
+```tsx
+const handleGenerate = async () => {
+  const chat: Message[] = [
+    { role: 'system', content: 'You are a helpful assistant' },
+    { role: 'user', content: 'What is the meaning of life?' }
+  ];
+
+  // Chat completion
+  await llm.generate(chat);
+  console.log('Llama says:', llm.response);
+};
+```
+
+## :calling: Examples
+
+We currently host a few example [apps](https://github.com/software-mansion/react-native-executorch/tree/main/apps) demonstrating use cases of our library:
+
+- `llm` - Chat application showcasing use of LLMs
+- `speech-to-text` - Whisper and Moonshine models ready for transcription tasks
+- `computer-vision` - Computer vision related tasks
+- `text-embeddings` - Computing text representations for semantic search
+
+If you would like to run demo app, navigate to its project directory and install dependencies with:
+
+```bash
+yarn
+```
+
+Then, depending on the platform, choose either iOS or Android:
+
+```bash
+yarn expo run:< ios | android >
+```
+
+> [!WARNING]  
+> Running LLMs requires a significant amount of RAM. If you are encountering unexpected app crashes, try to increase the amount of RAM allocated to the emulator.
+
+## :balance_scale: License
+
+This library is licensed under [The MIT License](./LICENSE).
+
+## :soon: What's next?
+
+To learn about our upcoming plans and developments, please visit our [milestones](https://github.com/software-mansion/react-native-executorch/milestones).
+
+## React Native ExecuTorch is created by Software Mansion
+
+Since 2012, [Software Mansion](https://swmansion.com) is a software agency with experience in building web and mobile apps. We are Core React Native Contributors and experts in dealing with all kinds of React Native issues. We can help you build your next dream product – [Hire us](https://swmansion.com/contact/projects?utm_source=react-native-executorch&utm_medium=readme).
+
+[![swm](https://logo.swmansion.com/logo?color=white&variant=desktop&width=150&tag=react-native-executorch-github 'Software Mansion')](https://swmansion.com)

package/common/rnexecutorch/models/speech_to_text/SpeechToText.cpp

@@ -1,4 +1,3 @@
-#include <rnexecutorch/models/speech_to_text/MoonshineStrategy.h>
 #include <rnexecutorch/models/speech_to_text/SpeechToText.h>
 #include <rnexecutorch/models/speech_to_text/WhisperStrategy.h>
 #include <stdexcept>
@@ -19,11 +18,9 @@ SpeechToText::SpeechToText(const std::string &encoderPath,
 void SpeechToText::initializeStrategy() {
   if (modelName == "whisper") {
     strategy = std::make_unique<WhisperStrategy>();
-  } else if (modelName == "moonshine") {
-    strategy = std::make_unique<MoonshineStrategy>();
   } else {
     throw std::runtime_error("Unsupported STT model: " + modelName +
-                             ". Only 'whisper' and 'moonshine' are supported.");
+                             ". Only 'whisper' is supported.");
   }
 }
 
@@ -40,7 +37,8 @@ void SpeechToText::encode(std::span<float> waveform) {
   encoderOutput = result.get().at(0);
 }
 
-int64_t SpeechToText::decode(std::vector<int64_t> prevTokens) {
+std::shared_ptr<OwningArrayBuffer>
+SpeechToText::decode(std::vector<int64_t> prevTokens) {
   if (encoderOutput.isNone()) {
     throw std::runtime_error("Empty encodings on decode call, make sure to "
                              "call encode() prior to decode()!");
@@ -49,9 +47,6 @@ int64_t SpeechToText::decode(std::vector<int64_t> prevTokens) {
   const auto prevTokensTensor = strategy->prepareTokenInput(prevTokens);
 
   const auto decoderMethod = strategy->getDecoderMethod();
-  // BEWARE!!!
-  // Moonshine will fail with invalid input if you pass large tokens i.e.
-  // Whisper's BOS/EOS
   const auto decoderResult =
       decoder_->execute(decoderMethod, {prevTokensTensor, encoderOutput});
 
@@ -63,8 +58,7 @@ int64_t SpeechToText::decode(std::vector<int64_t> prevTokens) {
 
   const auto decoderOutputTensor = decoderResult.get().at(0).toTensor();
   const auto innerDim = decoderOutputTensor.size(1);
-  return strategy->extractOutputToken(decoderOutputTensor.const_data_ptr(),
-                                      innerDim);
+  return strategy->extractOutputToken(decoderOutputTensor);
 }
 
 } // namespace rnexecutorch

package/common/rnexecutorch/models/speech_to_text/SpeechToText.h

@@ -18,7 +18,7 @@ public:
                         const std::string &modelName,
                         std::shared_ptr<react::CallInvoker> callInvoker);
   void encode(std::span<float> waveform);
-  int64_t decode(std::vector<int64_t> prevTokens);
+  std::shared_ptr<OwningArrayBuffer> decode(std::vector<int64_t> prevTokens);
 
 private:
   const std::string modelName;

package/common/rnexecutorch/models/speech_to_text/SpeechToTextStrategy.h

@@ -1,6 +1,7 @@
 #pragma once
 
 #include "executorch/extension/tensor/tensor_ptr.h"
+#include <rnexecutorch/host_objects/JSTensorViewOut.h>
 #include <span>
 #include <vector>
 
@@ -19,8 +20,8 @@ public:
 
   virtual std::string getDecoderMethod() const = 0;
 
-  virtual int64_t extractOutputToken(const void *outputPtr,
-                                     size_t innerDim) const = 0;
+  virtual std::shared_ptr<OwningArrayBuffer> extractOutputToken(
+      const executorch::aten::Tensor &decoderOutputTensor) const = 0;
 };
 
 } // namespace rnexecutorch

package/common/rnexecutorch/models/speech_to_text/WhisperStrategy.cpp

@@ -29,10 +29,22 @@ WhisperStrategy::prepareTokenInput(const std::vector<int64_t> &prevTokens) {
   return make_tensor_ptr(std::move(tensorSizes), std::move(tokens32));
 }
 
-int64_t WhisperStrategy::extractOutputToken(const void *outputPtr,
-                                            size_t innerDim) const {
-  const auto *data = static_cast<const int32_t *>(outputPtr);
-  return static_cast<int64_t>(data[innerDim - 1]);
+std::shared_ptr<OwningArrayBuffer> WhisperStrategy::extractOutputToken(
+    const executorch::aten::Tensor &decoderOutputTensor) const {
+  const auto innerDim = decoderOutputTensor.size(1);
+  const auto dictSize = decoderOutputTensor.size(2);
+  auto outputNumel = decoderOutputTensor.numel();
+  auto dataPtr =
+      static_cast<const float *>(decoderOutputTensor.const_data_ptr()) +
+      (innerDim - 1) * dictSize;
+
+  std::span<const float> modelOutput(dataPtr, outputNumel / innerDim);
+  auto createBuffer = [](const auto &data, size_t size) {
+    auto buffer = std::make_shared<OwningArrayBuffer>(size);
+    std::memcpy(buffer->data(), data, size);
+    return buffer;
+  };
+  return createBuffer(modelOutput.data(), modelOutput.size_bytes());
 }
 
 } // namespace rnexecutorch

package/common/rnexecutorch/models/speech_to_text/WhisperStrategy.h

@@ -14,8 +14,8 @@ public:
 
   std::string getDecoderMethod() const override { return "forward"; }
 
-  int64_t extractOutputToken(const void *outputPtr,
-                             size_t innerDim) const override;
+  std::shared_ptr<OwningArrayBuffer> extractOutputToken(
+      const executorch::aten::Tensor &decoderOutputTensor) const override;
 
 private:
   std::vector<float> preprocessedData;

package/lib/Error.d.ts

@@ -0,0 +1,30 @@
+export declare enum ETError {
+    UndefinedError = 101,
+    ModuleNotLoaded = 102,
+    FileWriteFailed = 103,
+    ModelGenerating = 104,
+    LanguageNotSupported = 105,
+    InvalidModelSource = 255,
+    MultilingualConfiguration = 160,
+    MissingDataChunk = 161,
+    StreamingNotStarted = 162,
+    Ok = 0,
+    Internal = 1,
+    InvalidState = 2,
+    EndOfMethod = 3,
+    NotSupported = 16,
+    NotImplemented = 17,
+    InvalidArgument = 18,
+    InvalidType = 19,
+    OperatorMissing = 20,
+    NotFound = 32,
+    MemoryAllocationFailed = 33,
+    AccessFailed = 34,
+    InvalidProgram = 35,
+    InvalidExternalData = 36,
+    OutOfResources = 37,
+    DelegateInvalidCompatibility = 48,
+    DelegateMemoryAllocationFailed = 49,
+    DelegateInvalidHandle = 50
+}
+export declare const getError: (e: unknown | ETError | Error) => string;

package/lib/Error.js

@@ -0,0 +1,50 @@
+export var ETError;
+(function (ETError) {
+    // React-native-ExecuTorch errors
+    ETError[ETError["UndefinedError"] = 101] = "UndefinedError";
+    ETError[ETError["ModuleNotLoaded"] = 102] = "ModuleNotLoaded";
+    ETError[ETError["FileWriteFailed"] = 103] = "FileWriteFailed";
+    ETError[ETError["ModelGenerating"] = 104] = "ModelGenerating";
+    ETError[ETError["LanguageNotSupported"] = 105] = "LanguageNotSupported";
+    ETError[ETError["InvalidModelSource"] = 255] = "InvalidModelSource";
+    // SpeechToText errors
+    ETError[ETError["MultilingualConfiguration"] = 160] = "MultilingualConfiguration";
+    ETError[ETError["MissingDataChunk"] = 161] = "MissingDataChunk";
+    ETError[ETError["StreamingNotStarted"] = 162] = "StreamingNotStarted";
+    // ExecuTorch mapped errors
+    // Based on: https://github.com/pytorch/executorch/blob/main/runtime/core/error.h
+    // System errors
+    ETError[ETError["Ok"] = 0] = "Ok";
+    ETError[ETError["Internal"] = 1] = "Internal";
+    ETError[ETError["InvalidState"] = 2] = "InvalidState";
+    ETError[ETError["EndOfMethod"] = 3] = "EndOfMethod";
+    // Logical errors
+    ETError[ETError["NotSupported"] = 16] = "NotSupported";
+    ETError[ETError["NotImplemented"] = 17] = "NotImplemented";
+    ETError[ETError["InvalidArgument"] = 18] = "InvalidArgument";
+    ETError[ETError["InvalidType"] = 19] = "InvalidType";
+    ETError[ETError["OperatorMissing"] = 20] = "OperatorMissing";
+    // Resource errors
+    ETError[ETError["NotFound"] = 32] = "NotFound";
+    ETError[ETError["MemoryAllocationFailed"] = 33] = "MemoryAllocationFailed";
+    ETError[ETError["AccessFailed"] = 34] = "AccessFailed";
+    ETError[ETError["InvalidProgram"] = 35] = "InvalidProgram";
+    ETError[ETError["InvalidExternalData"] = 36] = "InvalidExternalData";
+    ETError[ETError["OutOfResources"] = 37] = "OutOfResources";
+    // Delegate errors
+    ETError[ETError["DelegateInvalidCompatibility"] = 48] = "DelegateInvalidCompatibility";
+    ETError[ETError["DelegateMemoryAllocationFailed"] = 49] = "DelegateMemoryAllocationFailed";
+    ETError[ETError["DelegateInvalidHandle"] = 50] = "DelegateInvalidHandle";
+})(ETError || (ETError = {}));
+export const getError = (e) => {
+    if (typeof e === 'number') {
+        return ETError[e] ?? ETError[ETError.UndefinedError];
+    }
+    // try to extract number from message (can contain false positives)
+    const error = e;
+    const errorCode = parseInt(error.message, 10);
+    if (Number.isNaN(errorCode)) {
+        return error.message;
+    }
+    return ETError[errorCode] ?? ETError[ETError.UndefinedError];
+};

package/lib/constants/directories.d.ts

@@ -0,0 +1 @@
+export declare const RNEDirectory: string;

package/lib/constants/directories.js

@@ -0,0 +1,2 @@
+import { documentDirectory } from 'expo-file-system';
+export const RNEDirectory = `${documentDirectory}react-native-executorch/`;

package/lib/constants/llmDefaults.d.ts

@@ -0,0 +1,6 @@
+import { ChatConfig, Message } from '../types/llm';
+export declare const DEFAULT_SYSTEM_PROMPT = "You are a knowledgeable, efficient, and direct AI assistant. Provide concise answers, focusing on the key information needed. Offer suggestions tactfully when appropriate to improve outcomes. Engage in productive collaboration with the user. Don't return too much text.";
+export declare const DEFAULT_STRUCTURED_OUTPUT_PROMPT: (structuredOutputSchema: string) => string;
+export declare const DEFAULT_MESSAGE_HISTORY: Message[];
+export declare const DEFAULT_CONTEXT_WINDOW_LENGTH = 5;
+export declare const DEFAULT_CHAT_CONFIG: ChatConfig;

package/lib/constants/llmDefaults.js

@@ -0,0 +1,16 @@
+export const DEFAULT_SYSTEM_PROMPT = "You are a knowledgeable, efficient, and direct AI assistant. Provide concise answers, focusing on the key information needed. Offer suggestions tactfully when appropriate to improve outcomes. Engage in productive collaboration with the user. Don't return too much text.";
+export const DEFAULT_STRUCTURED_OUTPUT_PROMPT = (structuredOutputSchema) => `The output should be formatted as a JSON instance that conforms to the JSON schema below.
+
+As an example, for the schema {"properties": {"foo": {"title": "Foo", "description": "a list of strings", "type": "array", "items": {"type": "string"}}}, "required": ["foo"]}
+the object {"foo": ["bar", "baz"]} is a well-formatted instance of the schema. The object {"properties": {"foo": ["bar", "baz"]}} is not well-formatted.
+
+Here is the output schema:
+${structuredOutputSchema}
+`;
+export const DEFAULT_MESSAGE_HISTORY = [];
+export const DEFAULT_CONTEXT_WINDOW_LENGTH = 5;
+export const DEFAULT_CHAT_CONFIG = {
+    systemPrompt: DEFAULT_SYSTEM_PROMPT,
+    initialMessageHistory: DEFAULT_MESSAGE_HISTORY,
+    contextWindowLength: DEFAULT_CONTEXT_WINDOW_LENGTH,
+};