react-native-executorch 0.4.8 → 0.5.0

package/third-party/ios/ExecutorchLib/ExecutorchLib/runner/text_prefiller.cpp

@@ -0,0 +1,91 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+// Given a text prompt, encode it using tokenizer and prefill the KV cache of a
+// LLM.
+
+#include "text_prefiller.h"
+
+namespace executorch {
+namespace extension {
+namespace llm {
+
+TextPrefiller::TextPrefiller(TextDecoderRunner *text_decoder_runner,
+                             bool use_kv_cache, bool enable_parallel_prefill)
+    : text_decoder_runner_(text_decoder_runner), use_kv_cache_(use_kv_cache),
+      enable_parallel_prefill_(enable_parallel_prefill) {}
+
+::executorch::runtime::Result<uint64_t>
+TextPrefiller::prefill(std::vector<uint64_t> &prompt_tokens,
+                       int64_t &start_pos) {
+  ET_CHECK_MSG(!prompt_tokens.empty(), "Prompt cannot be null");
+  if (!text_decoder_runner_->is_method_loaded()) {
+    ET_CHECK_OK_OR_RETURN_ERROR(text_decoder_runner_->load());
+  }
+  // enable_parallel_prefill_ maybe set even when not using kv cache
+  // When kv cache is not used, start pos is ignored
+  int32_t num_prompt_tokens = prompt_tokens.size();
+
+  // store the token
+  uint64_t cur_token;
+  if (enable_parallel_prefill_ || !use_kv_cache_) {
+    // initialize tensor wrappers
+    auto tokens = from_blob(prompt_tokens.data(), {1, num_prompt_tokens},
+                            executorch::aten::ScalarType::Long);
+
+    auto start_pos_tensor =
+        from_blob(&start_pos, {1}, executorch::aten::ScalarType::Long);
+
+    auto outputs_res = text_decoder_runner_->step(tokens, start_pos_tensor);
+
+    ET_CHECK_OK_OR_RETURN_ERROR(outputs_res.error());
+    ET_LOG(Info, "Prefill token result numel(): %zu",
+           outputs_res.get().numel());
+
+    start_pos += num_prompt_tokens;
+    cur_token = text_decoder_runner_->logits_to_token(outputs_res.get());
+  } else {           // sequential prefill
+    int64_t pos = 0; // position in the sequence
+    // NOLINTNEXTLINE(facebook-hte-ParameterUncheckedArrayBounds)
+    cur_token = prompt_tokens[0];
+
+    // initialize tensor wrappers
+    auto tokens =
+        from_blob(&cur_token, {1, 1}, executorch::aten::ScalarType::Long);
+
+    auto start_pos_tensor =
+        from_blob(&start_pos, {1}, executorch::aten::ScalarType::Long);
+
+    // run the first token and get back logits tensor. Assuming the first token
+    // is bos so don't callback.
+    auto logits_tensor =
+        ET_UNWRAP(text_decoder_runner_->step(tokens, start_pos_tensor));
+
+    pos += 1; // start the loop from index 1
+    start_pos += 1;
+
+    while (pos < num_prompt_tokens) {
+      // Run the model
+      // NOLINTNEXTLINE(facebook-hte-ParameterUncheckedArrayBounds)
+      cur_token = prompt_tokens[pos];
+
+      logits_tensor =
+          ET_UNWRAP(text_decoder_runner_->step(tokens, start_pos_tensor));
+
+      pos++;
+      start_pos++;
+    }
+
+    cur_token = text_decoder_runner_->logits_to_token(logits_tensor);
+  }
+  return cur_token;
+}
+
+} // namespace llm
+} // namespace extension
+} // namespace executorch

package/third-party/ios/ExecutorchLib/ExecutorchLib/runner/text_prefiller.h

@@ -0,0 +1,51 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+// Given a text prompt, encode it using tokenizer and prefill the KV cache of a
+// LLM.
+
+#pragma once
+
+#include "text_decoder_runner.h"
+
+namespace executorch {
+namespace extension {
+namespace llm {
+
+class ET_EXPERIMENTAL TextPrefiller {
+public:
+  TextPrefiller(TextDecoderRunner *text_decoder_runner, bool use_kv_cache_,
+                bool enable_parallel_prefill);
+  /**
+   * Prefill an LLM Module with the given text input.
+   * @param prompt_tokens The text prompt tokens to the LLM Module. Encoded by
+   * tokenizer.
+   * @param start_pos The starting position in KV cache of the input in the LLM
+   * Module.
+   * @return The next token of the LLM Module after prefill.
+   */
+  ::executorch::runtime::Result<uint64_t>
+  prefill(std::vector<uint64_t> &prompt_tokens, int64_t &start_pos);
+
+private:
+  TextDecoderRunner *text_decoder_runner_;
+  bool use_kv_cache_;
+  bool enable_parallel_prefill_;
+};
+
+} // namespace llm
+} // namespace extension
+} // namespace executorch
+
+namespace torch {
+namespace executor {
+// TODO(T197294990): Remove these deprecated aliases once all users have moved
+// to the new `::executorch` namespaces.
+using ::executorch::extension::llm::TextPrefiller;
+} // namespace executor
+} // namespace torch

package/third-party/ios/ExecutorchLib/ExecutorchLib/runner/text_token_generator.h

@@ -0,0 +1,162 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+// Generate tokens in a loop.
+#pragma once
+
+#include "stats.h"
+#include "text_decoder_runner.h"
+#include <executorch/extension/tensor/tensor.h>
+#include <iostream>
+#include <tokenizers-cpp/tokenizers_cpp.h>
+
+namespace executorch {
+namespace extension {
+namespace llm {
+
+class ET_EXPERIMENTAL TextTokenGenerator {
+public:
+  TextTokenGenerator(tokenizers::Tokenizer *tokenizer,
+                     TextDecoderRunner *text_decoder_runner, bool use_kv_cache,
+                     std::unique_ptr<std::unordered_set<uint64_t>> &&eos_ids,
+                     Stats *stats)
+      : tokenizer_(tokenizer), text_decoder_runner_(text_decoder_runner),
+        eos_ids_(std::move(eos_ids)), use_kv_cache_(use_kv_cache),
+        stats_(stats) {}
+
+  /**
+   * Token generation loop.
+   * @param tokens prompt tokens as well as the first token generated by
+   * prefill.
+   * @param start_pos the start position of the new tokens, based on how many
+   * prompt tokens is prefilled.
+   * @param seq_len the total sequence length, including the prompt tokens, next
+   * token from prefill and new tokens.
+   * @param token_callback what to do after a token is generated.
+   * @return how many tokens are generated.
+   */
+  inline ::executorch::runtime::Result<int64_t>
+  generate(std::vector<uint64_t> tokens, int64_t start_pos, int32_t seq_len,
+           std::function<void(const std::string &)> token_callback) {
+    ET_CHECK_MSG(!tokens.empty(),
+                 "Token generation loop shouldn't take empty tokens");
+    int64_t pos = start_pos; // position in the sequence
+
+    std::vector<uint64_t> token_data; // allocate space for the tokens
+    std::vector<executorch::aten::SizesType> token_shape;
+
+    // Token after prefill
+    uint64_t cur_token = tokens.back();
+    uint64_t prev_token;
+    // cache to keep tokens if they were decoded into illegal character
+    std::vector<int32_t> token_cache;
+    // if first token after prefill was part of multi-token character we need to
+    // add this to cache here
+    if (tokenizer_->Decode(
+            std::vector<int32_t>{static_cast<int32_t>(cur_token)}) == "�") {
+      token_cache.push_back(static_cast<int32_t>(cur_token));
+    }
+
+    if (use_kv_cache_) {
+      // hard code these to size 1 as kv cache is locked to static size right
+      // now.
+      token_data = {cur_token};
+      token_shape = {1, 1};
+    } else {
+      token_data = tokens;
+      token_shape = {1, static_cast<int>(tokens.size())};
+    }
+
+    // initialize tensor wrappers
+    auto tokens_managed = from_blob(token_data.data(), token_shape,
+                                    executorch::aten::ScalarType::Long);
+    auto start_pos_managed =
+        from_blob(&pos, {1}, executorch::aten::ScalarType::Long);
+
+    should_stop_ = false;
+
+    // Generate our tokens
+    while (pos < seq_len - 1) {
+      // Run the model
+      auto logits_res =
+          text_decoder_runner_->step(tokens_managed, start_pos_managed);
+
+      ET_CHECK_OK_OR_RETURN_ERROR(logits_res.error());
+      executorch::aten::Tensor &logits_tensor = logits_res.get();
+
+      prev_token = cur_token;
+
+      stats_->on_sampling_begin();
+      cur_token = text_decoder_runner_->logits_to_token(logits_tensor);
+      stats_->on_sampling_end();
+
+      pos++;
+
+      if (use_kv_cache_) {
+        // update the token tensor. token_data will not be empty.
+        // NOLINTNEXTLINE(facebook-hte-LocalUncheckedArrayBounds)
+        token_data[0] = cur_token;
+      } else {
+        // push it to the back
+        token_data.push_back(cur_token);
+        ET_CHECK_OK_OR_RETURN_ERROR(resize_tensor_ptr(
+            tokens_managed, {1, static_cast<int>(token_data.size())}));
+      }
+
+      // print the tokens as string, decode it with the Tokenizer object
+      token_cache.push_back(static_cast<int32_t>(cur_token));
+      const std::string cache_decoded = tokenizer_->Decode(token_cache);
+
+      if (cache_decoded != "�" && cache_decoded != " �") {
+        token_callback(cache_decoded);
+        token_cache.clear();
+      }
+
+      if (should_stop_) {
+        break;
+      }
+
+      // data-dependent terminating condition: we have n_eos_ number of EOS
+      if (eos_ids_->find(cur_token) != eos_ids_->end()) {
+        printf("\n");
+        ET_LOG(Info, "\nReached to the end of generation");
+        break;
+      }
+    }
+    return pos - start_pos;
+  }
+
+  /**
+   * Stop the generation loop.
+   */
+  inline void stop() { should_stop_ = true; }
+
+private:
+  tokenizers::Tokenizer *tokenizer_;
+  TextDecoderRunner *text_decoder_runner_;
+  std::unique_ptr<std::unordered_set<uint64_t>> eos_ids_;
+  bool use_kv_cache_;
+
+  // state machine
+  bool should_stop_ = false;
+
+  // stats
+  Stats *stats_;
+};
+
+} // namespace llm
+} // namespace extension
+} // namespace executorch
+
+namespace torch {
+namespace executor {
+// TODO(T197294990): Remove these deprecated aliases once all users have moved
+// to the new `::executorch` namespaces.
+using ::executorch::extension::llm::TextTokenGenerator;
+} // namespace executor
+} // namespace torch

package/third-party/ios/ExecutorchLib/ExecutorchLib/runner/util.h

@@ -0,0 +1,108 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+#pragma once
+#include <cctype>
+#include <executorch/runtime/platform/compiler.h>
+#include <stdio.h>
+#include <time.h>
+#if defined(__linux__) || defined(__ANDROID__) || defined(__unix__)
+#include <sys/resource.h>
+#endif
+
+#define ET_UNWRAP_TOKENIZER(result__)                                          \
+  ({                                                                           \
+    auto tk_result__ = (result__);                                             \
+    if (!tk_result__.ok()) {                                                   \
+      ET_LOG(Error, "Tokenizers error code %d",                                \
+             static_cast<uint32_t>(tk_result__.error()));                      \
+      return ::executorch::runtime::Error::InvalidArgument;                    \
+    }                                                                          \
+    std::move(*tk_result__);                                                   \
+  })
+
+#define ET_CHECK_TK_OK_OR_RETURN_ERROR(result__, ...)                          \
+  ({                                                                           \
+    auto tk_result__ = (result__);                                             \
+    if (tk_result__ != ::tokenizers::Error::Ok) {                              \
+      ET_LOG(Error, "Tokenizer error: %d",                                     \
+             static_cast<uint32_t>(tk_result__));                              \
+      ET_CHECK_OK_OR_RETURN_ERROR(                                             \
+          ::executorch::runtime::Error::InvalidArgument, ##__VA_ARGS__);       \
+    }                                                                          \
+  })
+
+namespace executorch {
+namespace extension {
+namespace llm {
+
+ET_EXPERIMENTAL void inline safe_printf(const char *piece) {
+  // piece might be a raw byte token, and we only want to print printable chars
+  // or whitespace because some of the other bytes can be various control codes,
+  // backspace, etc.
+  if (piece == nullptr) {
+    return;
+  }
+  if (piece[0] == '\0') {
+    return;
+  }
+  if (piece[1] == '\0') {
+    unsigned char byte_val = piece[0];
+    if (!(isprint(byte_val) || isspace(byte_val))) {
+      return; // bad byte, don't print it
+    }
+  }
+  printf("%s", piece);
+}
+
+// ----------------------------------------------------------------------------
+// utilities: time
+
+ET_EXPERIMENTAL long inline time_in_ms() {
+  // return time in milliseconds, for benchmarking the model speed
+  struct timespec time;
+  clock_gettime(CLOCK_REALTIME, &time);
+  return time.tv_sec * 1000 + time.tv_nsec / 1000000;
+}
+
+// ----------------------------------------------------------------------------
+// utilities: memory usage
+
+// Returns the current RSS in bytes. Returns 0 if not supported.
+// RSS: Resident Set Size, the amount of memory currently in the RAM for this
+// process. These values are approximate, and are only used for logging
+// purposes.
+ET_EXPERIMENTAL size_t inline get_rss_bytes() {
+#if defined(__linux__) || defined(__ANDROID__) || defined(__unix__)
+  struct rusage r_usage;
+  if (getrusage(RUSAGE_SELF, &r_usage) == 0) {
+    return r_usage.ru_maxrss * 1024;
+  }
+#endif // __linux__ || __ANDROID__ || __unix__
+       // Unsupported platform like Windows, or getrusage() failed.
+       // __APPLE__ and __MACH__ are not supported because r_usage.ru_maxrss
+       // does not consistently return kbytes on macOS. On older versions of
+       // macOS, it returns bytes, but on newer versions it returns kbytes. Need
+       // to figure out when this changed.
+  return 0;
+}
+} // namespace llm
+} // namespace extension
+} // namespace executorch
+
+namespace torch {
+namespace executor {
+namespace util {
+// TODO(T197294990): Remove these deprecated aliases once all users have moved
+// to the new `::executorch` namespaces.
+using ::executorch::extension::llm::get_rss_bytes;
+using ::executorch::extension::llm::safe_printf;
+using ::executorch::extension::llm::time_in_ms;
+} // namespace util
+} // namespace executor
+} // namespace torch

package/third-party/ios/ExecutorchLib/ExecutorchLib/sampler/sampler.cpp

@@ -0,0 +1,193 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+// This is a modified version of https://github.com/karpathy/llama2.c.git
+// @lint-ignore-every LICENSELINT
+/**
+ * MIT License
+ *
+ * Copyright (c) 2023 Andrej
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "sampler.h"
+#include <algorithm>
+
+namespace executorch {
+namespace extension {
+namespace llm {
+
+// sampler stuff
+template <typename T> int32_t Sampler::sample_argmax(T *probabilities) {
+  // return the index that has the highest probability
+  int max_i = 0;
+  T max_p = probabilities[0];
+  for (int i = 1; i < vocab_size_; i++) {
+    if (probabilities[i] > max_p) {
+      max_i = i;
+      max_p = probabilities[i];
+    }
+  }
+  return max_i;
+}
+
+template <typename T>
+int32_t Sampler::sample_mult(T *probabilities, float coin) {
+  // sample index from probabilities (they must sum to 1!)
+  // coin is a random number in [0, 1), usually from random_f32()
+  T cdf = 0.0;
+  for (int i = 0; i < vocab_size_; i++) {
+    cdf += probabilities[i];
+    if (coin < cdf) {
+      return i;
+    }
+  }
+  return vocab_size_ - 1; // in case of rounding errors
+}
+
+template <typename T>
+int32_t Sampler::sample_topp(T *probabilities, float coin) {
+  // top-p sampling (or "nucleus sampling") samples from the smallest set of
+  // tokens that exceed probability topp. This way we never sample tokens that
+  // have very low probabilities and are less likely to go "off the rails".
+  // coin is a random number in [0, 1), usually from random_f32()
+  int n = vocab_size_;
+  int n0 = 0;
+  // quicksort indices in descending order of probabilities
+  // values smaller than (1 - topp) / (n - 1) cannot be part of the result
+  // so for efficiency we crop these out as candidates before sorting
+  std::unique_ptr<ProbIndex<T>[]> probindex =
+      std::make_unique<ProbIndex<T>[]>(vocab_size_);
+
+  const float cutoff = (1.0f - topp_) / (n - 1);
+  for (int i = 0; i < n; i++) {
+    if (probabilities[i] >= cutoff) {
+      probindex[n0].index = i;
+      probindex[n0].prob = probabilities[i];
+      n0++;
+    }
+  }
+
+  auto compare = [](const ProbIndex<T> &a, const ProbIndex<T> &b) {
+    return a.prob > b.prob;
+  };
+  std::sort(probindex.get(), probindex.get() + n0, compare);
+
+  // truncate the list where cumulative probability exceeds topp
+  T cumulative_prob = 0;
+  int last_idx = n0 - 1; // in case of rounding errors consider all elements
+  for (int i = 0; i < n0; i++) {
+    cumulative_prob += probindex[i].prob;
+    if (cumulative_prob > topp_) {
+      last_idx = i;
+      break; // we've exceeded topp by including last_idx
+    }
+  }
+
+  // sample from the truncated list
+  const T &r = coin * cumulative_prob;
+  T cdf = 0;
+  for (int i = 0; i <= last_idx; i++) {
+    cdf += probindex[i].prob;
+    if (r < cdf) {
+      return probindex[i].index;
+    }
+  }
+  return probindex[last_idx].index; // in case of rounding errors
+}
+
+Sampler::Sampler(int vocab_size, float temperature, float topp,
+                 unsigned long long rng_seed)
+    : vocab_size_(vocab_size),
+      inv_temperature_(static_cast<bool>(temperature) ? 1.0f / temperature : 0),
+      topp_(topp), rng_state_(rng_seed) {}
+
+template <typename T> static void softmax(T *x, int size) {
+  // find max value (for numerical stability)
+  T max_val = x[0];
+  for (int i = 1; i < size; i++) {
+    if (x[i] > max_val) {
+      max_val = x[i];
+    }
+  }
+  // exp and sum
+  T sum = 0;
+  for (int i = 0; i < size; i++) {
+    x[i] = expf(x[i] - max_val);
+    sum += x[i];
+  }
+  // normalize
+  for (int i = 0; i < size; i++) {
+    x[i] /= sum;
+  }
+}
+
+static unsigned int random_u32(unsigned long long *state) {
+  // xorshift rng: https://en.wikipedia.org/wiki/Xorshift#xorshift.2A
+  *state ^= *state >> 12;
+  *state ^= *state << 25;
+  *state ^= *state >> 27;
+  return (*state * 0x2545F4914F6CDD1Dull) >> 32;
+}
+
+static float random_f32(unsigned long long *state) { // random float32 in [0,1)
+  return (random_u32(state) >> 8) / 16777216.0f;
+}
+
+template <typename T> int32_t Sampler::sample(T *logits) {
+  // sample the token given the logits and some hyperparameters
+  int next;
+  if (inv_temperature_ == 0.0f) {
+    // greedy argmax sampling: take the token with the highest probability
+    next = sample_argmax(logits);
+  } else {
+    // apply the temperature to the logits
+    for (int q = 0; q < vocab_size_; q++) {
+      logits[q] *= inv_temperature_;
+    }
+    // apply softmax to the logits to get the probabilities for next token
+    softmax(logits, vocab_size_);
+    // flip a (float) coin (this is our source of entropy for sampling)
+    float coin = random_f32(&rng_state_);
+    // we sample from this distribution to get the next token
+    if (topp_ <= 0 || topp_ >= 1) {
+      // simply sample from the predicted probability distribution
+      next = sample_mult(logits, coin);
+    } else {
+      // top-p (nucleus) sampling, clamping the least likely tokens to zero
+      next = sample_topp(logits, coin);
+    }
+  }
+  return next;
+}
+
+template int32_t Sampler::sample<float>(float *logits);
+template int32_t Sampler::sample<exec_aten::Half>(exec_aten::Half *logits);
+template int32_t
+Sampler::sample<exec_aten::BFloat16>(exec_aten::BFloat16 *logits);
+
+} // namespace llm
+} // namespace extension
+} // namespace executorch

package/third-party/ios/ExecutorchLib/ExecutorchLib/sampler/sampler.h

@@ -0,0 +1,64 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+#pragma once
+
+#include <cctype>
+#include <cmath>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <memory>
+#ifdef USE_ATEN_LIB
+#include <torch/torch.h>
+#endif
+
+#include <executorch/runtime/core/exec_aten/exec_aten.h>
+
+namespace executorch {
+namespace extension {
+namespace llm {
+// A simple llama2 sampler.
+
+template <typename T> struct ProbIndex {
+  T prob;
+  int32_t index;
+}; // struct used when sorting probabilities during top-p sampling
+
+class Sampler {
+public:
+  Sampler(int32_t vocab_size, float temperature, float topp,
+          unsigned long long rng_seed);
+
+  template <typename T> int32_t sample(T *logits);
+
+private:
+  template <typename T> int32_t sample_topp(T *probabilities, float coin);
+  template <typename T> int32_t sample_mult(T *probabilities, float coin);
+  template <typename T> int32_t sample_argmax(T *probabilities);
+
+private:
+  int32_t vocab_size_;
+  // reciprocal of temperature, or 0 if temperature == 0.
+  float inv_temperature_;
+  float topp_;
+  unsigned long long rng_state_;
+};
+
+} // namespace llm
+} // namespace extension
+} // namespace executorch
+
+namespace torch {
+namespace executor {
+// TODO(T197294990): Remove these deprecated aliases once all users have moved
+// to the new `::executorch` namespaces.
+using ::executorch::extension::llm::ProbIndex;
+using ::executorch::extension::llm::Sampler;
+} // namespace executor
+} // namespace torch