react-native-executorch 0.4.10 → 0.5.0-nightly-6f4dd53-20251211

package/common/rnexecutorch/threads/GlobalThreadPool.h

@@ -0,0 +1,84 @@
+// GlobalThreadPool.h
+#pragma once
+
+#include <executorch/extension/threadpool/cpuinfo_utils.h>
+#include <memory>
+#include <mutex>
+#include <opencv2/opencv.hpp>
+#include <optional>
+#include <rnexecutorch/Log.h>
+#include <rnexecutorch/threads/HighPerformanceThreadPool.h>
+
+namespace rnexecutorch::threads {
+
+class GlobalThreadPool {
+public:
+  GlobalThreadPool() = delete;
+  GlobalThreadPool(const GlobalThreadPool &) = delete;
+  GlobalThreadPool &operator=(const GlobalThreadPool &) = delete;
+  GlobalThreadPool(GlobalThreadPool &&) = delete;
+  GlobalThreadPool &operator=(GlobalThreadPool &&) = delete;
+
+  static HighPerformanceThreadPool &get() {
+    if (!instance) {
+      initialize();
+    }
+    return *instance;
+  }
+
+  static void initialize(std::optional<uint32_t> numThreads = std::nullopt,
+                         ThreadConfig config = {}) {
+    std::call_once(initFlag, [&numThreads, config]() {
+      if (!numThreads) {
+        numThreads =
+            ::executorch::extension::cpuinfo::get_num_performant_cores();
+      }
+
+      numThreads = std::max(numThreads.value(), 2u);
+      log(rnexecutorch::LOG_LEVEL::Info, "Initializing global thread pool with",
+          numThreads, "threads");
+      instance = std::make_unique<HighPerformanceThreadPool>(numThreads.value(),
+                                                             config);
+      // Disable OpenCV's internal threading to prevent it from overriding our
+      // thread pool configuration, which would cause degraded performance
+      cv::setNumThreads(0);
+    });
+  }
+
+  // Convenience methods that mirror std::thread interface
+  template <typename Func, typename... Args>
+  static auto async(Func &&func, Args &&...args) {
+    return get().submit(std::forward<Func>(func), std::forward<Args>(args)...);
+  }
+
+  template <typename Func, typename... Args>
+  static auto async_high_priority(Func &&func, Args &&...args) {
+    return get().submitWithPriority(Priority::HIGH, std::forward<Func>(func),
+                                    std::forward<Args>(args)...);
+  }
+
+  // Fire and forget (like std::thread{}.detach())
+  template <typename Func, typename... Args>
+  static void detach(Func &&func, Args &&...args) {
+    get().submitDetached(std::forward<Func>(func), std::forward<Args>(args)...);
+  }
+
+  // Execute and wait (like std::thread{}.join())
+  template <typename Func, typename... Args>
+  static auto execute(Func &&func, Args &&...args) {
+    return get().execute(std::forward<Func>(func), std::forward<Args>(args)...);
+  }
+
+  static void shutdown() {
+    if (instance) {
+      instance->shutdown();
+      instance.reset();
+    }
+  }
+
+private:
+  inline static std::unique_ptr<HighPerformanceThreadPool> instance;
+  inline static std::once_flag initFlag;
+};
+
+} // namespace rnexecutorch::threads

package/common/rnexecutorch/threads/HighPerformanceThreadPool.h

@@ -0,0 +1,364 @@
+// HighPerformanceThreadPool.h
+#pragma once
+
+#include <algorithm>
+#include <atomic>
+#include <chrono>
+#include <condition_variable>
+#include <fstream>
+#include <functional>
+#include <future>
+#include <memory>
+#include <mutex>
+#include <pthread.h>
+#include <queue>
+#include <ranges>
+#include <sched.h>
+#include <sys/resource.h>
+#include <thread>
+#include <unistd.h>
+#include <vector>
+
+#include <executorch/extension/threadpool/cpuinfo_utils.h>
+#include <rnexecutorch/Log.h>
+
+#ifdef __APPLE__
+#include <sys/syscall.h>
+#endif
+
+#ifdef __ANDROID__
+#include <sys/types.h>
+#endif
+
+namespace rnexecutorch::threads {
+
+enum class Priority { LOW, NORMAL, HIGH, REALTIME };
+
+struct ThreadConfig {
+  bool pinToPerformanceCores{true};
+  std::string namePrefix{"RN_ET_Worker"};
+};
+
+class HighPerformanceThreadPool {
+public:
+  explicit HighPerformanceThreadPool(size_t numThreads = 1,
+                                     ThreadConfig cfg = ThreadConfig())
+      : config(std::move(cfg)) {
+
+#ifdef __ANDROID__
+    detectCPUTopology();
+    numThreads = std::max(std::min(numThreads, performanceCores.size()), 2lu);
+#endif
+
+    for (size_t i = 0; i < numThreads; i++) {
+      workers.emplace_back(&HighPerformanceThreadPool::workerThread, this, i);
+    }
+
+    log(LOG_LEVEL::Debug, "Thread pool initialized with", numThreads,
+        "workers.");
+  }
+
+  ~HighPerformanceThreadPool() { shutdown(); }
+
+  // Submit a task and get a future for the result
+  template <typename Func, typename... Args>
+  auto submit(Func &&func, Args &&...args)
+      -> std::future<decltype(func(args...))> {
+    return submitWithPriority(Priority::NORMAL, std::forward<Func>(func),
+                              std::forward<Args>(args)...);
+  }
+
+  // Submit a task with specific priority
+  template <typename Func, typename... Args>
+  auto submitWithPriority(Priority priority, Func &&func, Args &&...args)
+      -> std::future<decltype(func(args...))> {
+
+    using ReturnType = decltype(func(args...));
+
+    // Create a packaged task
+    auto boundFunc =
+        std::bind(std::forward<Func>(func), std::forward<Args>(args)...);
+    auto task = std::make_unique<Task<decltype(boundFunc), ReturnType>>(
+        std::move(boundFunc));
+    auto future = task->getFuture();
+
+    // Add to queue
+    {
+      std::scoped_lock lock(queueMutex);
+
+      if (!running) {
+        throw std::runtime_error("Thread pool is shutting down");
+      }
+
+      WorkItem item(std::move(task), priority,
+                    std::chrono::steady_clock::now());
+
+      taskQueue.push(std::move(item));
+    }
+
+    condition.notify_one();
+    return future;
+  }
+
+  // Execute a task and wait for result
+  template <typename Func, typename... Args>
+  auto execute(Func &&func, Args &&...args) -> decltype(func(args...)) {
+    auto future = submit(std::forward<Func>(func), std::forward<Args>(args)...);
+    return future.get();
+  }
+
+  // Fire and forget task
+  template <typename Func, typename... Args>
+  void submitDetached(Func &&func, Args &&...args) {
+    submit(std::forward<Func>(func), std::forward<Args>(args)...);
+    // Future is destroyed, task still runs
+  }
+
+  void shutdown() {
+    if (!running.exchange(false)) {
+      return;
+    }
+
+    condition.notify_all();
+
+    for (auto &worker : workers) {
+      if (worker.joinable()) {
+        worker.join();
+      }
+    }
+  }
+
+private:
+  // Task wrapper that can hold any callable
+  class ITask {
+  public:
+    virtual ~ITask() = default;
+    virtual void execute() = 0;
+  };
+
+  template <typename Func, typename Result> class Task : public ITask {
+  public:
+    Task(Func &&f) : func(std::forward<Func>(f)) {}
+
+    void execute() override {
+      try {
+        if constexpr (std::is_void_v<Result>) {
+          func();
+          promise.set_value();
+        } else {
+          promise.set_value(func());
+        }
+      } catch (...) {
+        promise.set_exception(std::current_exception());
+      }
+    }
+
+    std::future<Result> getFuture() { return promise.get_future(); }
+
+  private:
+    Func func;
+    std::promise<Result> promise;
+  };
+
+  class WorkItem {
+  public:
+    WorkItem() = default;
+    WorkItem(std::unique_ptr<ITask> task, Priority priority,
+             std::chrono::steady_clock::time_point enqueueTime)
+        : task(std::move(task)), priority(priority), enqueueTime(enqueueTime) {}
+
+    std::unique_ptr<ITask> task;
+
+    bool operator<(const WorkItem &other) const {
+      return priority != other.priority ? priority < other.priority
+                                        : enqueueTime > other.enqueueTime;
+    }
+
+  private:
+    Priority priority;
+    std::chrono::steady_clock::time_point enqueueTime;
+  };
+
+  // Thread pool state
+  std::vector<std::thread> workers;
+  std::priority_queue<WorkItem> taskQueue;
+  std::mutex queueMutex;
+  std::condition_variable condition;
+  std::atomic<bool> running{true};
+  std::atomic<size_t> activeWorkers{0};
+  std::atomic<size_t> totalTasksProcessed{0};
+
+#ifdef __ANDROID__
+  // Performance cores
+  std::vector<int32_t> performanceCores;
+  std::vector<int32_t> efficiencyCores;
+#endif
+
+  // Configuration
+  ThreadConfig config;
+
+  void detectCPUTopology() {
+#ifdef __ANDROID__
+    struct CoreInfo {
+      int32_t id;
+      int64_t maxFreq;
+    };
+
+    std::vector<CoreInfo> cores;
+    const auto numOfCores = std::thread::hardware_concurrency();
+
+    for (int32_t i = 0; std::cmp_less(i, numOfCores); ++i) {
+      std::string path = "/sys/devices/system/cpu/cpu" + std::to_string(i) +
+                         "/cpufreq/cpuinfo_max_freq";
+      std::ifstream file(path);
+      if (!file.good()) {
+        break;
+      }
+
+      CoreInfo info;
+      info.id = i;
+      file >> info.maxFreq;
+      cores.push_back(info);
+    }
+
+    if (cores.empty()) {
+      log(LOG_LEVEL::Debug, "Could not detect CPU topology");
+      return;
+    }
+
+    // Sort by frequency
+    std::ranges::sort(cores, [](const CoreInfo &a, const CoreInfo &b) {
+      return a.maxFreq > b.maxFreq;
+    });
+
+    // Classify cores
+    const auto numOfPerfCores =
+        ::executorch::extension::cpuinfo::get_num_performant_cores();
+
+    constexpr float kKiloToGigaRatio = 1e6;
+    for (int32_t i = 0; i < cores.size(); ++i) {
+      if (i < numOfPerfCores) {
+        performanceCores.push_back(cores[i].id);
+        log(LOG_LEVEL::Debug, "Performance core:", cores[i].id, "(",
+            cores[i].maxFreq / kKiloToGigaRatio, "GHz)");
+      } else {
+        efficiencyCores.push_back(cores[i].id);
+        log(LOG_LEVEL::Debug, "Efficiency core:", cores[i].id, "(",
+            cores[i].maxFreq / kKiloToGigaRatio, "GHz)");
+      }
+    }
+#endif
+  }
+
+#ifdef __ANDROID__
+  inline uint64_t getCurrentThreadId() { return gettid(); }
+#endif
+
+  inline void setCurrentThreadName(const std::string &name) {
+#ifdef __ANDROID__
+    pthread_setname_np(pthread_self(), name.c_str());
+#elif defined(__APPLE__)
+    pthread_setname_np(name.c_str());
+#endif
+  }
+
+  void configureThread(uint32_t workerIndex) {
+    std::string threadName = config.namePrefix + std::to_string(workerIndex);
+    setCurrentThreadName(threadName.c_str());
+
+#ifdef __ANDROID__
+    if (config.pinToPerformanceCores && !performanceCores.empty()) {
+      setCPUAffinity();
+    }
+#endif
+
+    setThreadPriority();
+
+    log(LOG_LEVEL::Debug, "Worker", workerIndex,
+        "configured:", threadName.c_str());
+  }
+
+  void setCPUAffinity() {
+    // AFAIK it is not possible on iOS
+#ifdef __ANDROID__
+    if (performanceCores.empty()) {
+      log(LOG_LEVEL::Error, "No cores specified for affinity setting");
+      return;
+    }
+
+    cpu_set_t cpuset;
+    CPU_ZERO(&cpuset);
+
+    for (int32_t core : performanceCores) {
+      CPU_SET(core, &cpuset);
+    }
+
+    pid_t tid = getCurrentThreadId();
+    log(LOG_LEVEL::Debug, "Thread id", tid);
+    if (sched_setaffinity(tid, sizeof(cpuset), &cpuset) == 0) {
+      log(LOG_LEVEL::Debug, "Thread pinned to cores:", performanceCores);
+    } else {
+      log(LOG_LEVEL::Debug, "Failed to set CPU affinity (error:", errno,
+          "). Continuing without affinity.");
+    }
+#endif
+  }
+
+  void setThreadPriority() {
+    // pthread_setschedparam doesn't work on android because permissions reasons
+    // and in general does not provide visible improvements on iOS
+
+    // Set nice value as fallback or additional priority boost
+    constexpr int nice_value = 0;
+    if (setpriority(PRIO_PROCESS, 0, nice_value) != 0) {
+      log(LOG_LEVEL::Debug, "Failed to set nice value");
+    } else {
+      log(LOG_LEVEL::Debug, "Set nice value", nice_value);
+    }
+  }
+
+  void processTask(const WorkItem &item) {
+    activeWorkers++;
+
+    try {
+      item.task->execute();
+    } catch (const std::exception &e) {
+      log(LOG_LEVEL::Error, "Task failed:", e.what());
+      activeWorkers--;
+      throw;
+    }
+
+    activeWorkers--;
+    totalTasksProcessed++;
+  }
+
+  void workerThread(int workerIndex) {
+    configureThread(workerIndex);
+
+    while (running) {
+      WorkItem item;
+
+      {
+        std::unique_lock<std::mutex> lock(queueMutex);
+        condition.wait(lock, [this] { return !taskQueue.empty() || !running; });
+
+        if (!running && taskQueue.empty()) {
+          break;
+        }
+
+        if (!taskQueue.empty()) {
+          item = std::move(const_cast<WorkItem &>(taskQueue.top()));
+          taskQueue.pop();
+        } else {
+          continue;
+        }
+      }
+
+      processTask(item);
+    }
+
+    log(LOG_LEVEL::Debug, "Worker", workerIndex, "shutting down");
+  }
+};
+
+} // namespace rnexecutorch::threads

package/common/rnexecutorch/threads/utils/ThreadUtils.h

@@ -0,0 +1,29 @@
+#pragma once
+
+#include <executorch/extension/threadpool/cpuinfo_utils.h>
+#include <executorch/extension/threadpool/threadpool.h>
+#include <rnexecutorch/Log.h>
+
+namespace rnexecutorch::threads::utils {
+
+void unsafeSetupThreadPool(uint32_t num_of_cores = 0) {
+  auto num_of_perf_cores =
+      ::executorch::extension::cpuinfo::get_num_performant_cores();
+  log(LOG_LEVEL::Info, "Detected ", num_of_perf_cores, " performant cores");
+  // setting num_of_cores to floor(num_of_perf_cores / 2) + 1) because
+  // depending on cpu arch as when possible we want to leave at least 2
+  // performant cores for other tasks (setting more actually results in drop
+  // of performance). For older devices (i.e. samsung s22) resolves to 3
+  // cores, and for newer ones (like OnePlus 12) resolves to 4, which when
+  // benchmarked gives highest throughput. For iPhones they usually have 2
+  // performance cores
+  auto _num_of_cores = num_of_cores
+                           ? num_of_cores
+                           : static_cast<uint32_t>(num_of_perf_cores / 2) + 1;
+  const auto threadpool = ::executorch::extension::threadpool::get_threadpool();
+  threadpool->_unsafe_reset_threadpool(_num_of_cores);
+  log(LOG_LEVEL::Info, "Configuring xnnpack for",
+      threadpool->get_thread_count(), "threads");
+}
+
+} // namespace rnexecutorch::threads::utils

package/common/runner/arange_util.cpp

@@ -0,0 +1,44 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+#include "arange_util.h"
+
+namespace torch::executor::native {
+#define ET_ARANGE_IMPL(ctx, start, numel, step, out, op_name)                  \
+  ET_SWITCH_REALHBF16_TYPES(out.scalar_type(), ctx, op_name, CTYPE, [&]() {    \
+    auto out_data = out.mutable_data_ptr<CTYPE>();                             \
+    for (executorch::aten::SizesType i = 0; i < numel; ++i) {                  \
+      out_data[i] = static_cast<CTYPE>(start + i * step);                      \
+    }                                                                          \
+  })
+
+executorch::aten::SizesType compute_arange_out_size(double start, double end,
+                                                    double step) {
+  executorch::aten::SizesType numel =
+      static_cast<executorch::aten::SizesType>(std::ceil((end - start) / step));
+
+  ET_CHECK_MSG(numel >= 0,
+               "numel should be non-negative, but got (%" PRId64
+               "). start (%f), end (%f), step (%f)",
+               static_cast<int64_t>(numel), start, end, step);
+  return numel;
+}
+
+void arange_out_impl(KernelRuntimeContext &ctx, double start, double end,
+                     double step, Tensor &out) {
+  (void)ctx;
+  executorch::aten::SizesType numel = compute_arange_out_size(start, end, step);
+  ET_ARANGE_IMPL(ctx, start, numel, step, out, "arange.start_out");
+}
+
+void arange_out_impl(KernelRuntimeContext &ctx, double end, Tensor &out) {
+  (void)ctx;
+  ET_ARANGE_IMPL(ctx, 0.0, end, 1.0, out, "arange.out");
+}
+
+} // namespace torch::executor::native

package/common/runner/arange_util.h

@@ -0,0 +1,37 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+#pragma once
+
+#include "kernel_includes.h"
+
+namespace torch::executor::native {
+
+executorch::aten::SizesType compute_arange_out_size(double start, double end,
+                                                    double step);
+
+inline executorch::aten::SizesType compute_arange_out_size(double end) {
+  return compute_arange_out_size(0.0, end, 1.0);
+}
+
+void arange_out_impl(KernelRuntimeContext &ctx, double start, double end,
+                     double step, Tensor &out);
+
+void arange_out_impl(KernelRuntimeContext &ctx, double end, Tensor &out);
+
+inline void arange_out_impl(double start, double end, double step,
+                            Tensor &out) {
+  KernelRuntimeContext ctx;
+  arange_out_impl(ctx, start, end, step, out);
+}
+
+inline void arange_out_impl(double end, Tensor &out) {
+  KernelRuntimeContext ctx;
+  arange_out_impl(ctx, 0.0, end, 1.0, out);
+}
+} // namespace torch::executor::native

package/common/runner/constants.h

@@ -0,0 +1,28 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+#pragma once
+// constants for LLM runtime
+namespace executorch::extension::llm {
+
+// Runtime metadata key constants
+inline constexpr auto kEnableDynamicShape = "enable_dynamic_shape";
+inline constexpr auto kBosId = "get_bos_id";
+inline constexpr auto kEosIds = "get_eos_ids";
+inline constexpr auto kMaxSeqLen = "get_max_seq_len";
+inline constexpr auto kMaxContextLen = "get_max_context_len";
+inline constexpr auto kVocabSize = "get_vocab_size";
+inline constexpr auto kUseKVCache = "use_kv_cache";
+inline constexpr auto kUseSDPAWithKVCache = "use_sdpa_with_kv_cache";
+
+// Multimodal method name conventions
+inline constexpr auto kVisionEncoderMethod = "vision_encoder";
+inline constexpr auto kAudioEncoderMethod = "audio_encoder";
+inline constexpr auto kTokenEmbeddingMethod = "token_embedding";
+inline constexpr auto kTextModelMethod = "text_decoder";
+
+} // namespace executorch::extension::llm