react-native-executorch 0.4.9 → 0.5.0-nightly-6f4dd53-20251211

package/third-party/include/executorch/extension/threadpool/threadpool.h

@@ -0,0 +1,95 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+#pragma once
+
+#include <functional>
+#include <memory>
+#include <mutex>
+
+#include <pthreadpool/pthreadpool.h>
+
+namespace executorch::extension::threadpool {
+
+class ThreadPool final {
+public:
+  explicit ThreadPool(size_t thread_count = 0);
+  ~ThreadPool() = default;
+
+  // Make threadpool non copyable
+  // Non-copyable: threadpool cannot be copied because it will
+  // effectively require cloning of threadpool.
+  // Cloning can be done by just calling create_thread_pool.
+  ThreadPool(const ThreadPool &) = delete;
+  ThreadPool &operator=(const ThreadPool &) = delete;
+
+  // Make threadpool non-movable.
+  ThreadPool(ThreadPool &&) = delete;
+  ThreadPool &operator=(ThreadPool &&) = delete;
+
+  size_t get_thread_count() const;
+
+  /**
+   * INTERNAL: Resets the threadpool by creating a new threadpool with requested
+   * # of threads. This is not a thread safe call. When calling this method,
+   * threads of the threadpool might be doing some work. Some other code may
+   * also be holding on to the threadpool pointer, that is no longer valid. This
+   * is a private API, which will later be replaced by something that allows
+   * creating of threadpool with requested size and use such a threadpool with
+   * backend delegates, custom ops or optimized lib.
+   * For Meta internal use, there is
+   * executorch::extension::threadpool::UseNThreadsThreadPoolGuard API that
+   * provides a safer way to select a subset of threads, from threadpool, to run
+   * the model on.
+   */
+  bool _unsafe_reset_threadpool(uint32_t num_threads);
+
+  /**
+   * Run, in parallel, function fn(task_id) over task_id in range [0, range).
+   * This function is blocking.  All input is processed by the time it returns.
+   * NoThreadPoolGuard (see threadpool_guard.h) can used to disable use of
+   * multiple threads with the scope of the guard When NoThreadPoolGuard is not
+   * used all calls to run method are serialized.
+   */
+  void run(const std::function<void(size_t)> &fn, size_t range);
+
+private:
+  friend pthreadpool_t get_pthreadpool();
+
+private:
+  // This mutex is used inside get_thread_count API but it is not really needed
+  // since data members of ThreadPool objects are not really mutable.
+  // TODO(kimishpatel): Figure out if we will allow set_num_threads API, in
+  // which case this mutex will be useful. Otherwise remove it.
+  mutable std::mutex mutex_;
+  std::unique_ptr<pthreadpool, decltype(&pthreadpool_destroy)> threadpool_;
+};
+
+/**
+ * Returns the singleton instance of ThreadPool for ATen/TH multithreading.
+ */
+ThreadPool *get_threadpool();
+
+/**
+ * Returns the underlying pthreadpool instance used by the implementation of
+ * ThreadPool returned by `get_threadpool()`. Only for use in external libraries
+ * so as to unify threading across internal (i.e. ATen, etc.) and external (e.g.
+ * NNPACK, QNNPACK, XNNPACK) use cases.
+ */
+pthreadpool_t get_pthreadpool();
+
+} // namespace executorch::extension::threadpool
+
+namespace torch::executorch::threadpool { // DEPRECATED
+// TODO(T197294990): Remove these deprecated aliases once all users have moved
+// to the new `::executorch` namespaces. Note that threadpool incorrectly used
+// the namespace `torch::executorch` instead of `torch::executor`.
+using ::executorch::extension::threadpool::get_pthreadpool; // DEPRECATED
+using ::executorch::extension::threadpool::get_threadpool;  // DEPRECATED
+using ::executorch::extension::threadpool::ThreadPool;      // DEPRECATED
+} // namespace torch::executorch::threadpool

package/third-party/include/executorch/runtime/backend/backend_execution_context.h

@@ -0,0 +1,71 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+#pragma once
+
+#include <executorch/runtime/core/event_tracer.h>
+#include <executorch/runtime/core/memory_allocator.h>
+
+namespace executorch {
+namespace ET_RUNTIME_NAMESPACE {
+
+/**
+ * BackendExecutionContext will be used to inject run time context.
+ */
+class BackendExecutionContext final {
+public:
+  BackendExecutionContext(EventTracer *event_tracer = nullptr,
+                          MemoryAllocator *temp_allocator = nullptr,
+                          const char *method_name = nullptr)
+      : event_tracer_(event_tracer), temp_allocator_(temp_allocator),
+        method_name_(method_name) {}
+
+  /**
+   * Returns a pointer to an instance of EventTracer to do profiling/debugging
+   * logging inside the delegate backend. Users will need access to this pointer
+   * to use any of the event tracer APIs.
+   */
+  EventTracer *event_tracer() { return event_tracer_; }
+
+  /**
+   * Returns a pointer to the address allocated by temp allocator. This
+   * allocator will be reset after every delegate call during execution.
+   */
+  void *allocate(size_t size,
+                 size_t alignment = MemoryAllocator::kDefaultAlignment) {
+    // TODO(chenlai): depends on the need, we may expose more functionality for
+    // memory allocation.
+    return temp_allocator_->allocate(size, alignment);
+  }
+
+  /**
+   * Returns the temp allocator. This allocator will be reset every instruction.
+   */
+  MemoryAllocator *get_temp_allocator() { return temp_allocator_; }
+
+  /**
+   * Get the name of the executing method from the ExecuTorch runtime.
+   */
+  const char *get_method_name() const { return method_name_; }
+
+private:
+  EventTracer *event_tracer_ = nullptr;
+  MemoryAllocator *temp_allocator_ = nullptr;
+  const char *method_name_ = nullptr;
+};
+
+} // namespace ET_RUNTIME_NAMESPACE
+} // namespace executorch
+
+namespace torch {
+namespace executor {
+// TODO(T197294990): Remove these deprecated aliases once all users have moved
+// to the new `::executorch` namespaces.
+using ::executorch::ET_RUNTIME_NAMESPACE::BackendExecutionContext;
+} // namespace executor
+} // namespace torch

package/third-party/include/executorch/runtime/backend/backend_init_context.h

@@ -0,0 +1,84 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+#pragma once
+#include <executorch/runtime/core/event_tracer.h>
+#include <executorch/runtime/core/memory_allocator.h>
+#include <executorch/runtime/core/named_data_map.h>
+
+#ifdef __GNUC__
+// Disable -Wdeprecated-declarations, as some builds use 'Werror'.
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
+#endif
+
+namespace executorch {
+namespace ET_RUNTIME_NAMESPACE {
+/**
+ * BackendInitContext will be used to inject runtime info for to initialize
+ * delegate.
+ */
+class BackendInitContext final {
+public:
+  explicit BackendInitContext(MemoryAllocator *runtime_allocator,
+                              EventTracer *event_tracer = nullptr,
+                              const char *method_name = nullptr,
+                              const NamedDataMap *named_data_map = nullptr)
+      : runtime_allocator_(runtime_allocator),
+#ifdef ET_EVENT_TRACER_ENABLED
+        event_tracer_(event_tracer),
+#else
+        event_tracer_(nullptr),
+#endif
+        method_name_(method_name), named_data_map_(named_data_map) {
+  }
+
+  /** Get the runtime allocator passed from Method. It's the same runtime
+   * executor used by the standard executor runtime and the life span is the
+   * same as the model.
+   */
+  MemoryAllocator *get_runtime_allocator() { return runtime_allocator_; }
+
+  /**
+   * Returns a pointer (null if not installed) to an instance of EventTracer to
+   * do profiling/debugging logging inside the delegate backend. Users will need
+   * access to this pointer to use any of the event tracer APIs.
+   */
+  EventTracer *event_tracer() { return event_tracer_; }
+
+  /** Get the loaded method name from ExecuTorch runtime. Usually it's
+   * "forward", however, if there are multiple methods in the .pte file, it can
+   * be different. One example is that we may have prefill and decode methods in
+   * the same .pte file. In this case, when client loads "prefill" method, the
+   * `get_method_name` function will return "prefill", when client loads
+   * "decode" method, the `get_method_name` function will return "decode".
+   */
+  const char *get_method_name() const { return method_name_; }
+
+  /** Get the named data map from ExecuTorch runtime.
+   * This provides a way for backends to retrieve data blobs by key.
+   */
+  const NamedDataMap *get_named_data_map() const { return named_data_map_; }
+
+private:
+  MemoryAllocator *runtime_allocator_ = nullptr;
+  EventTracer *event_tracer_ = nullptr;
+  const char *method_name_ = nullptr;
+  const NamedDataMap *named_data_map_ = nullptr;
+};
+
+} // namespace ET_RUNTIME_NAMESPACE
+} // namespace executorch
+
+namespace torch {
+namespace executor {
+// TODO(T197294990): Remove these deprecated aliases once all users have moved
+// to the new `::executorch` namespaces.
+using ::executorch::ET_RUNTIME_NAMESPACE::BackendInitContext;
+} // namespace executor
+} // namespace torch

package/third-party/include/executorch/runtime/backend/backend_option_context.h

@@ -0,0 +1,34 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+#pragma once
+#include <executorch/runtime/core/event_tracer.h>
+#include <executorch/runtime/core/memory_allocator.h>
+#include <executorch/runtime/core/named_data_map.h>
+
+namespace executorch {
+namespace ET_RUNTIME_NAMESPACE {
+/**
+ * BackendOptionContext will be used to inject runtime info for to initialize
+ * delegate.
+ */
+class BackendOptionContext final {
+public:
+  explicit BackendOptionContext() {}
+};
+
+} // namespace ET_RUNTIME_NAMESPACE
+} // namespace executorch
+
+namespace torch {
+namespace executor {
+// TODO(T197294990): Remove these deprecated aliases once all users have moved
+// to the new `::executorch` namespaces.
+using ::executorch::ET_RUNTIME_NAMESPACE::BackendOptionContext;
+} // namespace executor
+} // namespace torch

package/third-party/include/executorch/runtime/backend/interface.h

@@ -0,0 +1,227 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+#pragma once
+
+#include <cstring>
+
+#include <executorch/runtime/backend/backend_execution_context.h>
+#include <executorch/runtime/backend/backend_init_context.h>
+#include <executorch/runtime/backend/backend_option_context.h>
+#include <executorch/runtime/backend/options.h>
+#include <executorch/runtime/core/array_ref.h>
+#include <executorch/runtime/core/error.h>
+#include <executorch/runtime/core/evalue.h>
+#include <executorch/runtime/core/freeable_buffer.h>
+#include <executorch/runtime/core/memory_allocator.h>
+#include <executorch/runtime/core/named_data_map.h>
+#include <executorch/runtime/core/result.h>
+#include <executorch/runtime/platform/compiler.h>
+
+namespace executorch {
+namespace ET_RUNTIME_NAMESPACE {
+
+struct SizedBuffer {
+  void *buffer;
+  size_t nbytes; // number of bytes of buffer
+};
+
+struct CompileSpec {
+  const char *key;   // spec key
+  SizedBuffer value; // spec value
+};
+
+/**
+ * An opaque handle managed by a backend. Typically points to a backend-private
+ * class/struct.
+ */
+using DelegateHandle = void;
+
+class BackendInterface {
+public:
+  virtual ~BackendInterface() = 0;
+
+  /**
+   * Returns true if the backend is available to process delegation calls.
+   */
+  ET_NODISCARD virtual bool is_available() const = 0;
+
+  /**
+   * Responsible to further process (compile/transform/optimize) the compiled
+   * unit that was produced, ahead-of-time, as well as perform any backend
+   * initialization to ready it for execution. This method is called every time
+   * the ExecuTorch program is initialized. Consequently, this is the place to
+   * perform any backend initialization as well as transformations,
+   * optimizations, and even compilation that depend on the target device. As
+   * such, it is strongly encouraged to push as much processing as possible to
+   * the ahead-of-time processing.
+   *
+   * @param[in] processed An opaque (to ExecuTorch) backend-specific compiled
+   *     unit from the preprocessor. Can contain anything the backend needs to
+   *     execute the equivalent semantics of the passed-in Module and its
+   *     method. Often passed unmodified to `execute()` as a `DelegateHandle`,
+   *     unless it needs further processing at init time to be fully executable.
+   *     If the data is not needed after init(), calling processed->Free() can
+   *     reclaim its memory.
+   * @param[in] compile_specs The exact same compiler specification that
+   *     was used ahead-of-time to produce `processed`.
+   *
+   * @returns On success, an opaque handle representing the the method
+   *     implemented by the delegate. This handle is passed to `execute()` and
+   *     `destroy()`, and the memory it points to is owned by the backend.
+   *     Typically points to a backend-private class/struct.
+   * @returns On error, returns an error code other than Error::Ok. If the
+   *     compiled unit (the preprocessed result from ahead of time) is not
+   *     compatible with the current backend runtime, return the error code
+   *     Error::DelegateInvalidCompatibility. Other backend delegate
+   *     specific error codes can be found in error.h.
+   */
+  ET_NODISCARD virtual Result<DelegateHandle *>
+  init(BackendInitContext &context, FreeableBuffer *processed,
+       ArrayRef<CompileSpec> compile_specs) const = 0;
+
+  /**
+   * Responsible for executing the given method’s handle, as it was produced
+   * by compile.
+   *
+   * @param[in] handle An opaque handle returned by `init()`. Usually a backend
+   *     executable unit. This executable unit should be ready to execute the
+   *     delegate blobs.
+   * @param[in] args The method’s inputs and outputs.
+   * @retval Error::Ok if successful.
+   */
+  ET_NODISCARD virtual Error execute(BackendExecutionContext &context,
+                                     DelegateHandle *handle,
+                                     Span<EValue *> args) const = 0;
+
+  /**
+   * Responsible update the backend status, if any. The backend options are
+   * passed in by users, and the backend can update its internal status based on
+   * the options.
+   *
+   * @param[in] context Runtime context if any. Currently it's not used.
+   * @param[in] args A list of BackendOptions passed in by users.
+   * @retval Error::Ok if successful.
+   */
+  ET_NODISCARD virtual Error
+  set_option(__ET_UNUSED BackendOptionContext &context,
+             const executorch::runtime::Span<BackendOption> &backend_options) {
+    return Error::Ok;
+  };
+
+  /**
+   * Responsible update the backend status, if any. The backend options are
+   * passed in by users, and the backend can update its internal status based on
+   * the options.
+   *
+   * @param[in] context Runtime context if any. Currently it's not used.
+   * @param[in] args A list of BackendOptions passed in by users, that will be
+   * filled by the backend
+   * @retval Error::Ok if successful.
+   */
+  ET_NODISCARD virtual Error
+  get_option(__ET_UNUSED BackendOptionContext &context,
+             executorch::runtime::Span<BackendOption> &backend_options) {
+    return Error::Ok;
+  };
+
+  /**
+   * Responsible for destroying a handle, if it's required for some backend.
+   * It may be needed for some backends. For example, resources associated with
+   * this handle needs to be released. This method is called when the execution
+   * plan is destroyed (i.e., the program is out of its lifespan).
+   *
+   * @param[in] handle The handle to be destroyed. An opaque handle returned by
+   *     `init()`.
+   */
+  virtual void destroy(ET_UNUSED DelegateHandle *handle) const {}
+};
+
+/**
+ * Returns the corresponding object pointer for a given string name.
+ * The mapping is populated using register_backend method.
+ *
+ * @param[in] name Name of the user-defined backend delegate.
+ * @retval Pointer to the appropriate object that implements BackendInterface.
+ *         Nullptr if it can't find anything with the given name.
+ */
+BackendInterface *get_backend_class(const char *name);
+
+/**
+ * A named instance of a backend.
+ */
+struct Backend {
+  /// The name of the backend. Must match the string used in the PTE file.
+  const char *name;
+  /// The instance of the backend to use when loading and executing programs.
+  BackendInterface *backend;
+};
+
+/**
+ * Registers the Backend object (i.e. string name and BackendInterface pair) so
+ * that it could be called via the name during the runtime.
+ *
+ * @param[in] backend Backend object
+ * @retval Error code representing whether registration was successful.
+ */
+ET_NODISCARD Error register_backend(const Backend &backend);
+
+/**
+ * Returns the number of registered backends.
+ */
+size_t get_num_registered_backends();
+
+/**
+ * Returns the backend name at the given index.
+ */
+Result<const char *> get_backend_name(size_t index);
+
+/**
+ * Sets backend options for a specific backend.
+ *
+ * @param backend_name The name of the backend to set options for
+ * @param backend_options The backend option list containing the options
+ * to set
+ * @return Error::Ok on success, Error::NotFound if backend is not found, or
+ * other error codes on failure
+ */
+Error set_option(
+    const char *backend_name,
+    const executorch::runtime::Span<executorch::runtime::BackendOption>
+        backend_options);
+
+/**
+ * Retrieves backend options for a specific backend.
+ *
+ * @param backend_name The name of the backend to get options from
+ * @param backend_options The backend option objects that will be filled with
+ * the populated values from the backend
+ * @return Error::Ok on success, Error::NotFound if backend is not found, or
+ * other error codes on failure
+ */
+Error get_option(const char *backend_name,
+                 executorch::runtime::Span<executorch::runtime::BackendOption>
+                     backend_options);
+
+} // namespace ET_RUNTIME_NAMESPACE
+} // namespace executorch
+
+namespace torch {
+namespace executor {
+// TODO(T197294990): Remove these deprecated aliases once all users have moved
+// to the new `::executorch` namespaces.
+using ::executorch::ET_RUNTIME_NAMESPACE::Backend;
+using ::executorch::ET_RUNTIME_NAMESPACE::CompileSpec;
+using ::executorch::ET_RUNTIME_NAMESPACE::DelegateHandle;
+using ::executorch::ET_RUNTIME_NAMESPACE::get_backend_class;
+using ::executorch::ET_RUNTIME_NAMESPACE::register_backend;
+using ::executorch::ET_RUNTIME_NAMESPACE::SizedBuffer;
+using PyTorchBackendInterface =
+    ::executorch::ET_RUNTIME_NAMESPACE::BackendInterface;
+} // namespace executor
+} // namespace torch