react-native-executorch 0.5.5 → 0.5.7

package/android/src/main/cpp/CMakeLists.txt

@@ -1,11 +1,12 @@
 cmake_minimum_required(VERSION 3.13)
 
 file(GLOB_RECURSE ANDROID_CPP_SOURCES CONFIGURE_DEPENDS "${ANDROID_CPP_DIR}/*.cpp")
-file(GLOB_RECURSE COMMON_CPP_SOURCES CONFIGURE_DEPENDS "${COMMON_CPP_DIR}/*.cpp" "${COMMON_CPP_DIR}/*.c")
+file(GLOB_RECURSE COMMON_CPP_SOURCES CONFIGURE_DEPENDS "${COMMON_CPP_DIR}/*.cpp")
+file(GLOB_RECURSE COMMON_C_SOURCES CONFIGURE_DEPENDS "${COMMON_CPP_DIR}/*.c")
 file(GLOB_RECURSE TEST_CPP_SOURCES "${COMMON_CPP_DIR}/rnexecutorch/tests/*.cpp")
 list(REMOVE_ITEM COMMON_CPP_SOURCES ${TEST_CPP_SOURCES})
 
-add_library(react-native-executorch SHARED ${ANDROID_CPP_SOURCES} ${COMMON_CPP_SOURCES})
+add_library(react-native-executorch SHARED ${ANDROID_CPP_SOURCES} ${COMMON_CPP_SOURCES} ${COMMON_C_SOURCES})
 
 find_package(ReactAndroid REQUIRED CONFIG)
 find_package(fbjni REQUIRED CONFIG)
@@ -41,23 +42,21 @@ add_library(executorch SHARED IMPORTED)
 set_target_properties(executorch PROPERTIES
                       IMPORTED_LOCATION "${LIBS_DIR}/executorch/${ANDROID_ABI}/libexecutorch.so")
 
-                      
+
 if(ANDROID_ABI STREQUAL "arm64-v8a")
   target_compile_definitions(react-native-executorch PRIVATE ARCH_ARM64)
 
   # ------- pthreadpool -------
   add_library(pthreadpool SHARED IMPORTED)
-  
+
   set_target_properties(pthreadpool PROPERTIES
-                        IMPORTED_LOCATION "${LIBS_DIR}/pthreadpool/${ANDROID_ABI}/libpthreadpool.so"
-                        INTERFACE_INCLUDE_DIRECTORIES "${LIBS_DIR}/../../include/pthreadpool/")
-  
+                        IMPORTED_LOCATION "${LIBS_DIR}/pthreadpool/${ANDROID_ABI}/libpthreadpool.so")
+
   # ------- cpuinfo -------
   add_library(cpuinfo SHARED IMPORTED)
-  
+
   set_target_properties(cpuinfo PROPERTIES
-                        IMPORTED_LOCATION "${LIBS_DIR}/cpuinfo/${ANDROID_ABI}/libcpuinfo.so"
-                        INTERFACE_INCLUDE_DIRECTORIES "${LIBS_DIR}/../../include/cpuinfo/")
+                        IMPORTED_LOCATION "${LIBS_DIR}/cpuinfo/${ANDROID_ABI}/libcpuinfo.so")
   set(EXECUTORCH_LIBS
     "pthreadpool"
     "cpuinfo"
@@ -66,7 +65,7 @@ endif()
 
 # ------- OpenCV -------
 
-set(OPENCV_LIBS 
+set(OPENCV_LIBS
   "${LIBS_DIR}/opencv/${ANDROID_ABI}/libopencv_core.a"
   "${LIBS_DIR}/opencv/${ANDROID_ABI}/libopencv_features2d.a"
   "${LIBS_DIR}/opencv/${ANDROID_ABI}/libopencv_highgui.a"
@@ -85,15 +84,25 @@ elseif(ANDROID_ABI STREQUAL "x86_64")
   set(OPENCV_THIRD_PARTY_LIBS "")
 endif()
 
+# ------- tokenizers-cpp -------
+
+set(TOKENIZERS_LIBS
+  "${LIBS_DIR}/tokenizers-cpp/${ANDROID_ABI}/libtokenizers_c.a"
+  "${LIBS_DIR}/tokenizers-cpp/${ANDROID_ABI}/libtokenizers_cpp.a"
+  "${LIBS_DIR}/tokenizers-cpp/${ANDROID_ABI}/libsentencepiece.a"
+)
 # --------------
 
+target_link_options(react-native-executorch PRIVATE -fopenmp -static-openmp)
+
 target_link_libraries(
-  react-native-executorch 
-  ${LINK_LIBRARIES} 
+  react-native-executorch
+  ${LINK_LIBRARIES}
   ${RN_VERSION_LINK_LIBRARIES}
   ${OPENCV_LIBS}
   ${OPENCV_THIRD_PARTY_LIBS}
+  ${TOKENIZERS_LIBS}
   executorch
   ${EXECUTORCH_LIBS}
   z
-)
+)

package/common/rnexecutorch/RnExecutorchInstaller.cpp

@@ -12,12 +12,8 @@
 #include <rnexecutorch/models/speech_to_text/SpeechToText.h>
 #include <rnexecutorch/models/style_transfer/StyleTransfer.h>
 #include <rnexecutorch/models/vertical_ocr/VerticalOCR.h>
-
-#if defined(__ANDROID__) && defined(__aarch64__)
-#include <executorch/extension/threadpool/cpuinfo_utils.h>
-#include <executorch/extension/threadpool/threadpool.h>
-#include <rnexecutorch/Log.h>
-#endif
+#include <rnexecutorch/threads/GlobalThreadPool.h>
+#include <rnexecutorch/threads/utils/ThreadUtils.h>
 
 namespace rnexecutorch {
 
@@ -92,21 +88,8 @@ void RnExecutorchInstaller::injectJSIBindings(
       RnExecutorchInstaller::loadModel<models::speech_to_text::SpeechToText>(
           jsiRuntime, jsCallInvoker, "loadSpeechToText"));
 
-#if defined(__ANDROID__) && defined(__aarch64__)
-  auto num_of_perf_cores =
-      ::executorch::extension::cpuinfo::get_num_performant_cores();
-  log(LOG_LEVEL::Info, "Detected ", num_of_perf_cores, " performant cores");
-  // setting num_of_cores to floor(num_of_perf_cores / 2) + 1) because depending
-  // on cpu arch as when possible we want to leave at least 2 performant cores
-  // for other tasks (setting more actually results in drop of performance). For
-  // older devices (i.e. samsung s22) resolves to 3 cores, and for newer ones
-  // (like OnePlus 12) resolves to 4, which when benchamrked gives highest
-  // throughput.
-  auto num_of_cores = static_cast<uint32_t>(num_of_perf_cores / 2) + 1;
-  ::executorch::extension::threadpool::get_threadpool()
-      ->_unsafe_reset_threadpool(num_of_cores);
-  log(LOG_LEVEL::Info, "Configuring xnnpack for ", num_of_cores, " threads");
-#endif
+  threads::utils::unsafeSetupThreadPool();
+  threads::GlobalThreadPool::initialize();
 }
 
 } // namespace rnexecutorch

package/common/rnexecutorch/host_objects/ModelHostObject.h

@@ -19,6 +19,7 @@
 #include <rnexecutorch/models/llm/LLM.h>
 #include <rnexecutorch/models/ocr/OCR.h>
 #include <rnexecutorch/models/vertical_ocr/VerticalOCR.h>
+#include <rnexecutorch/threads/GlobalThreadPool.h>
 
 namespace rnexecutorch {
 
@@ -114,6 +115,19 @@ public:
           ModelHostObject<Model>, synchronousHostFunction<&Model::interrupt>,
           "interrupt"));
 
+      addFunctions(JSI_EXPORT_FUNCTION(
+          ModelHostObject<Model>,
+          synchronousHostFunction<&Model::getGeneratedTokenCount>,
+          "getGeneratedTokenCount"));
+
+      addFunctions(
+          JSI_EXPORT_FUNCTION(ModelHostObject<Model>,
+                              synchronousHostFunction<&Model::setCountInterval>,
+                              "setCountInterval"));
+
+      addFunctions(JSI_EXPORT_FUNCTION(
+          ModelHostObject<Model>,
+          synchronousHostFunction<&Model::setTimeInterval>, "setTimeInterval"));
       addFunctions(
           JSI_EXPORT_FUNCTION(ModelHostObject<Model>, unload, "unload"));
     }
@@ -195,58 +209,60 @@ public:
             // We need to dispatch a thread if we want the function to be
             // asynchronous. In this thread all accesses to jsi::Runtime need to
             // be done via the callInvoker.
-            std::thread([this, promise,
-                         argsConverted = std::move(argsConverted)]() {
-              try {
-                if constexpr (std::is_void_v<decltype(std::apply(
-                                  std::bind_front(FnPtr, model),
-                                  argsConverted))>) {
-                  // For void functions, just call the function and resolve with
-                  // undefined
-                  std::apply(std::bind_front(FnPtr, model),
-                             std::move(argsConverted));
-                  callInvoker->invokeAsync([promise](jsi::Runtime &runtime) {
-                    promise->resolve(jsi::Value::undefined());
-                  });
-                } else {
-                  // For non-void functions, capture the result and convert it
-                  auto result = std::apply(std::bind_front(FnPtr, model),
-                                           std::move(argsConverted));
-                  // The result is copied. It should either be quickly copiable,
-                  // or passed with a shared_ptr.
-                  callInvoker->invokeAsync(
-                      [promise, result](jsi::Runtime &runtime) {
-                        promise->resolve(jsi_conversion::getJsiValue(
-                            std::move(result), runtime));
-                      });
-                }
-              } catch (const std::runtime_error &e) {
-                // This catch should be merged with the next two
-                // (std::runtime_error and jsi::JSError inherits from
-                // std::exception) HOWEVER react native has broken RTTI which
-                // breaks proper exception type checking. Remove when the
-                // following change is present in our version:
-                // https://github.com/facebook/react-native/commit/3132cc88dd46f95898a756456bebeeb6c248f20e
-                callInvoker->invokeAsync([e = std::move(e), promise]() {
-                  promise->reject(e.what());
-                });
-                return;
-              } catch (const jsi::JSError &e) {
-                callInvoker->invokeAsync([e = std::move(e), promise]() {
-                  promise->reject(e.what());
-                });
-                return;
-              } catch (const std::exception &e) {
-                callInvoker->invokeAsync([e = std::move(e), promise]() {
-                  promise->reject(e.what());
+            threads::GlobalThreadPool::detach(
+                [this, promise, argsConverted = std::move(argsConverted)]() {
+                  try {
+                    if constexpr (std::is_void_v<decltype(std::apply(
+                                      std::bind_front(FnPtr, model),
+                                      argsConverted))>) {
+                      // For void functions, just call the function and resolve
+                      // with undefined
+                      std::apply(std::bind_front(FnPtr, model),
+                                 std::move(argsConverted));
+                      callInvoker->invokeAsync(
+                          [promise](jsi::Runtime &runtime) {
+                            promise->resolve(jsi::Value::undefined());
+                          });
+                    } else {
+                      // For non-void functions, capture the result and convert
+                      // it
+                      auto result = std::apply(std::bind_front(FnPtr, model),
+                                               std::move(argsConverted));
+                      // The result is copied. It should either be quickly
+                      // copiable, or passed with a shared_ptr.
+                      callInvoker->invokeAsync(
+                          [promise, result](jsi::Runtime &runtime) {
+                            promise->resolve(jsi_conversion::getJsiValue(
+                                std::move(result), runtime));
+                          });
+                    }
+                  } catch (const std::runtime_error &e) {
+                    // This catch should be merged with the next two
+                    // (std::runtime_error and jsi::JSError inherits from
+                    // std::exception) HOWEVER react native has broken RTTI
+                    // which breaks proper exception type checking. Remove when
+                    // the following change is present in our version:
+                    // https://github.com/facebook/react-native/commit/3132cc88dd46f95898a756456bebeeb6c248f20e
+                    callInvoker->invokeAsync([e = std::move(e), promise]() {
+                      promise->reject(e.what());
+                    });
+                    return;
+                  } catch (const jsi::JSError &e) {
+                    callInvoker->invokeAsync([e = std::move(e), promise]() {
+                      promise->reject(e.what());
+                    });
+                    return;
+                  } catch (const std::exception &e) {
+                    callInvoker->invokeAsync([e = std::move(e), promise]() {
+                      promise->reject(e.what());
+                    });
+                    return;
+                  } catch (...) {
+                    callInvoker->invokeAsync(
+                        [promise]() { promise->reject("Unknown error"); });
+                    return;
+                  }
                 });
-                return;
-              } catch (...) {
-                callInvoker->invokeAsync(
-                    [promise]() { promise->reject("Unknown error"); });
-                return;
-              }
-            }).detach();
           } catch (...) {
             promise->reject("Couldn't parse JS arguments in a native function");
           }

package/common/rnexecutorch/models/llm/LLM.cpp

@@ -1,7 +1,9 @@
 #include "LLM.h"
 
+#include <atomic>
 #include <executorch/extension/tensor/tensor.h>
 #include <filesystem>
+#include <rnexecutorch/threads/GlobalThreadPool.h>
 
 namespace rnexecutorch::models::llm {
 using namespace facebook;
@@ -49,10 +51,31 @@ void LLM::interrupt() {
   runner->stop();
 }
 
-std::size_t LLM::getMemoryLowerBound() const noexcept {
+size_t LLM::getGeneratedTokenCount() const noexcept {
+  if (!runner || !runner->is_loaded()) {
+    return 0;
+  }
+  return runner->stats_.num_generated_tokens;
+}
+
+size_t LLM::getMemoryLowerBound() const noexcept {
   return memorySizeLowerBound;
 }
 
+void LLM::setCountInterval(size_t countInterval) {
+  if (!runner || !runner->is_loaded()) {
+    throw std::runtime_error("Can't configure a model that's not loaded!");
+  }
+  runner->set_count_interval(countInterval);
+}
+
+void LLM::setTimeInterval(size_t timeInterval) {
+  if (!runner || !runner->is_loaded()) {
+    throw std::runtime_error("Can't configure a model that's not loaded!");
+  }
+  runner->set_time_interval(timeInterval);
+}
+
 void LLM::unload() noexcept { runner.reset(nullptr); }
 
 } // namespace rnexecutorch::models::llm

package/common/rnexecutorch/models/llm/LLM.h

@@ -21,7 +21,10 @@ public:
   void generate(std::string input, std::shared_ptr<jsi::Function> callback);
   void interrupt();
   void unload() noexcept;
-  std::size_t getMemoryLowerBound() const noexcept;
+  size_t getGeneratedTokenCount() const noexcept;
+  size_t getMemoryLowerBound() const noexcept;
+  void setCountInterval(size_t countInterval);
+  void setTimeInterval(size_t timeInterval);
 
 private:
   size_t memorySizeLowerBound;

package/common/rnexecutorch/models/speech_to_text/SpeechToText.cpp

@@ -90,8 +90,6 @@ void SpeechToText::stream(std::shared_ptr<jsi::Function> callback,
         });
   };
 
-  this->resetStreamState();
-
   this->isStreaming = true;
   while (this->isStreaming) {
     if (!this->readyToProcess ||
@@ -107,14 +105,13 @@ void SpeechToText::stream(std::shared_ptr<jsi::Function> callback,
 
   std::string committed = this->processor->finish();
   nativeCallback(committed, "", true);
+
+  this->resetStreamState();
 }
 
 void SpeechToText::streamStop() { this->isStreaming = false; }
 
 void SpeechToText::streamInsert(std::span<float> waveform) {
-  if (!this->isStreaming) {
-    throw std::runtime_error("Streaming is not started");
-  }
   this->processor->insertAudioChunk(waveform);
   this->readyToProcess = true;
 }

package/common/rnexecutorch/models/speech_to_text/SpeechToText.h

@@ -28,6 +28,7 @@ public:
   void streamInsert(std::span<float> waveform);
 
 private:
+  std::shared_ptr<react::CallInvoker> callInvoker;
   std::unique_ptr<BaseModel> encoder;
   std::unique_ptr<BaseModel> decoder;
   std::unique_ptr<TokenizerModule> tokenizer;
@@ -37,7 +38,6 @@ private:
   makeOwningBuffer(std::span<const float> vectorView) const;
 
   // Stream
-  std::shared_ptr<react::CallInvoker> callInvoker;
   std::unique_ptr<stream::OnlineASRProcessor> processor;
   bool isStreaming;
   bool readyToProcess;

package/common/rnexecutorch/threads/GlobalThreadPool.h

@@ -0,0 +1,79 @@
+// GlobalThreadPool.h
+#pragma once
+
+#include <executorch/extension/threadpool/cpuinfo_utils.h>
+#include <memory>
+#include <mutex>
+#include <optional>
+#include <rnexecutorch/Log.h>
+#include <rnexecutorch/threads/HighPerformanceThreadPool.h>
+
+namespace rnexecutorch::threads {
+
+class GlobalThreadPool {
+public:
+  GlobalThreadPool() = delete;
+  GlobalThreadPool(const GlobalThreadPool &) = delete;
+  GlobalThreadPool &operator=(const GlobalThreadPool &) = delete;
+  GlobalThreadPool(GlobalThreadPool &&) = delete;
+  GlobalThreadPool &operator=(GlobalThreadPool &&) = delete;
+
+  static HighPerformanceThreadPool &get() {
+    if (!instance) {
+      initialize();
+    }
+    return *instance;
+  }
+
+  static void initialize(std::optional<uint32_t> numThreads = std::nullopt,
+                         ThreadConfig config = {}) {
+    std::call_once(initFlag, [&numThreads, config]() {
+      if (!numThreads) {
+        numThreads =
+            ::executorch::extension::cpuinfo::get_num_performant_cores();
+      }
+
+      log(rnexecutorch::LOG_LEVEL::Info, "Initializing global thread pool with",
+          numThreads, "threads");
+      instance = std::make_unique<HighPerformanceThreadPool>(numThreads.value(),
+                                                             config);
+    });
+  }
+
+  // Convenience methods that mirror std::thread interface
+  template <typename Func, typename... Args>
+  static auto async(Func &&func, Args &&...args) {
+    return get().submit(std::forward<Func>(func), std::forward<Args>(args)...);
+  }
+
+  template <typename Func, typename... Args>
+  static auto async_high_priority(Func &&func, Args &&...args) {
+    return get().submitWithPriority(Priority::HIGH, std::forward<Func>(func),
+                                    std::forward<Args>(args)...);
+  }
+
+  // Fire and forget (like std::thread{}.detach())
+  template <typename Func, typename... Args>
+  static void detach(Func &&func, Args &&...args) {
+    get().submitDetached(std::forward<Func>(func), std::forward<Args>(args)...);
+  }
+
+  // Execute and wait (like std::thread{}.join())
+  template <typename Func, typename... Args>
+  static auto execute(Func &&func, Args &&...args) {
+    return get().execute(std::forward<Func>(func), std::forward<Args>(args)...);
+  }
+
+  static void shutdown() {
+    if (instance) {
+      instance->shutdown();
+      instance.reset();
+    }
+  }
+
+private:
+  inline static std::unique_ptr<HighPerformanceThreadPool> instance;
+  inline static std::once_flag initFlag;
+};
+
+} // namespace rnexecutorch::threads