react-native-executorch 0.7.0 → 0.7.2

package/third-party/include/executorch/extension/llm/tokenizers/include/pytorch/tokenizers/map_utils.h

@@ -0,0 +1,174 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+// @lint-ignore-every LICENSELINT
+
+#pragma once
+
+#include <algorithm>
+#include <memory>
+#include <string>
+#include <type_traits>
+#include <unordered_map>
+#include <vector>
+
+#include <pytorch/tokenizers/error.h>
+#include <pytorch/tokenizers/regex.h>
+#include <pytorch/tokenizers/result.h>
+#include <pytorch/tokenizers/string_integer_map.h>
+
+#include "re2/re2.h"
+
+namespace tokenizers {
+namespace detail {
+
+// Hash function for std::pair<uint64_t, uint64_t>
+struct PairHash {
+  std::size_t operator()(const std::pair<uint64_t, uint64_t> &p) const {
+    return std::hash<uint64_t>{}(p.first) ^
+           (std::hash<uint64_t>{}(p.second) << 1);
+  }
+};
+
+// Type alias for BPE merge map: (token_id_1, token_id_2) -> (rank,
+// merged_token_id)
+using MergeMap = std::unordered_map<std::pair<uint64_t, uint64_t>,
+                                    std::pair<uint64_t, uint64_t>, PairHash>;
+
+using TokenMap = StringIntegerMap<>;
+
+template <typename TToken, typename TRank>
+static Result<TokenMap>
+build_token_map(std::vector<std::pair<TToken, TRank>> container) {
+  static_assert(std::is_same_v<TToken, std::string> ||
+                    std::is_same_v<TToken, std::string_view>,
+                "TToken must be std::string or std::string_view");
+  static_assert(std::is_integral_v<TRank> && std::is_unsigned_v<TRank>,
+                "TRank must be an unsigned integer");
+
+  std::sort(container.begin(), container.end(),
+            [](const auto &a, const auto &b) { return a.first < b.first; });
+
+  auto duplicate_begin = std::unique(
+      container.begin(), container.end(),
+      [](const auto &a, const auto &b) { return a.first == b.first; });
+
+  TK_CHECK_OR_RETURN_ERROR(
+      duplicate_begin == container.end(), ParseFailure,
+      "duplicate token: %s rank: %llu", duplicate_begin->first.c_str(),
+      static_cast<unsigned long long>(duplicate_begin->second));
+
+  std::sort(container.begin(), container.end(),
+            [](const auto &a, const auto &b) { return a.second < b.second; });
+
+  duplicate_begin = std::unique(
+      container.begin(), container.end(),
+      [](const auto &a, const auto &b) { return a.second == b.second; });
+
+  TK_CHECK_OR_RETURN_ERROR(
+      duplicate_begin == container.end(), ParseFailure,
+      "duplicate rank: %llu"
+      " token: %s",
+      static_cast<unsigned long long>(duplicate_begin->second),
+      duplicate_begin->first.c_str());
+
+  return TokenMap(container);
+};
+
+template <typename TContainer, typename TTokenAccessor, typename TRankAccessor>
+static Result<TokenMap> build_token_map(const TContainer &container,
+                                        TTokenAccessor token_accessor,
+                                        TRankAccessor rank_accessor) {
+  using TokenType = std::invoke_result_t<TTokenAccessor, const TContainer &>;
+  using RankType = std::invoke_result_t<TRankAccessor, const TContainer &>;
+
+  static_assert(std::is_same_v<TokenType, std::string> ||
+                    std::is_same_v<TokenType, std::string_view>,
+                "TokenType must be std::string or std::string_view");
+  static_assert(std::is_integral_v<RankType> && std::is_unsigned_v<RankType>,
+                "RankType must be an unsigned integer");
+
+  std::vector<std::pair<TokenType, RankType>> pairs;
+  pairs.reserve(container.size());
+  for (const auto &value : container) {
+    pairs.emplace_back(token_accessor(value), rank_accessor(value));
+  }
+
+  return build_token_map(std::move(pairs));
+}
+
+// Utility function to build merge ranks map from merge rules
+template <typename TMergeMap>
+inline Result<TokenMap> build_merge_ranks_map(const TMergeMap &merge_map,
+                                              const TokenMap &token_map) {
+  // Static assertions to verify TMergeMap has the expected key and value types
+  using KeyType = typename TMergeMap::key_type;
+  using ValueType = typename TMergeMap::mapped_type;
+
+  static_assert(std::is_same_v<KeyType, std::pair<uint64_t, uint64_t>>,
+                "TMergeMap key type must be std::pair<uint64_t, uint64_t>");
+
+  static_assert(std::is_same_v<ValueType, std::pair<uint64_t, uint64_t>>,
+                "TMergeMap value type must be std::pair<uint64_t, uint64_t>");
+
+  // Use a map to handle duplicates - keep the lowest rank (highest priority)
+  std::unordered_map<std::string, uint64_t> unique_merge_ranks;
+
+  for (const auto &[pair, rank_and_id] : merge_map) {
+    uint64_t first_id = pair.first;
+    uint64_t second_id = pair.second;
+    uint64_t rank = rank_and_id.first;
+
+    // Get the token strings for the pair
+    auto first_token = token_map.tryGetString(first_id);
+    auto second_token = token_map.tryGetString(second_id);
+
+    if (first_token && second_token) {
+      std::string merged_token =
+          std::string(*first_token) + std::string(*second_token);
+
+      // Keep the entry with the lowest rank (highest priority in BPE)
+      auto it = unique_merge_ranks.find(merged_token);
+      if (it == unique_merge_ranks.end() || rank < it->second) {
+        unique_merge_ranks[merged_token] = rank;
+      }
+    }
+  }
+
+  // Convert to vector for buildTokenMap
+  std::vector<std::pair<std::string, uint64_t>> merge_rank_pairs;
+  merge_rank_pairs.reserve(unique_merge_ranks.size());
+
+  for (const auto &[token, rank] : unique_merge_ranks) {
+    merge_rank_pairs.emplace_back(token, rank);
+  }
+
+  return build_token_map(std::move(merge_rank_pairs));
+}
+
+inline Result<std::unique_ptr<IRegex>>
+build_special_token_regex(const TokenMap &special_token_map) {
+  std::string special_pattern;
+  const std::size_t count = special_token_map.size();
+
+  for (std::size_t i = 0; i < count; ++i) {
+    const auto &[token, _] = special_token_map.getElement(i);
+    if (!special_pattern.empty()) {
+      special_pattern += "|";
+    }
+    special_pattern += re2::RE2::QuoteMeta(std::string(token));
+  }
+
+  if (special_pattern.empty()) {
+    return static_cast<std::unique_ptr<IRegex>>(nullptr);
+  }
+  // Wrap pattern in parentheses for proper grouping
+  return create_regex("(" + special_pattern + ")");
+}
+
+} // namespace detail
+} // namespace tokenizers

package/third-party/include/executorch/extension/llm/tokenizers/include/pytorch/tokenizers/model.h

@@ -0,0 +1,151 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+// @lint-ignore-every LICENSELINT
+
+#pragma once
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include <nlohmann/json.hpp>
+#include <pytorch/tokenizers/map_utils.h>
+#include <pytorch/tokenizers/result.h>
+#include <pytorch/tokenizers/string_integer_map.h>
+
+namespace tokenizers {
+
+// -- Base ---------------------------------------------------------------------
+
+/**
+ * Abstract base class for tokenization models.
+ *
+ * A Model corresponds to the core logic that converts a piece of text (usually
+ * resulting from the pre-tokenization step) into a sequence of token IDs, and
+ * vice-versa.
+ *
+ * It encapsulates the vocabulary and the algorithm (e.g., BPE, WordPiece,
+ * Unigram).
+ */
+class Model {
+public:
+  using Ptr = std::shared_ptr<Model>;
+
+  virtual ~Model() = default;
+
+  /**
+   * Tokenizes a string piece into a sequence of token IDs.
+   *
+   * @param piece The input string to tokenize.
+   * @return A Result containing the vector of token IDs.
+   */
+  virtual Result<std::vector<uint64_t>>
+  tokenize(const std::string &piece) const = 0;
+
+  /**
+   * Converts a token ID to its string representation.
+   *
+   * @param token The token ID.
+   * @return A Result containing the string representation of the token.
+   */
+  virtual Result<std::string> id_to_piece(uint64_t token) const = 0;
+
+  /**
+   * Converts a string representation to its token ID.
+   *
+   * @param piece The string representation of the token.
+   * @return A Result containing the token ID.
+   */
+  virtual Result<uint64_t> piece_to_id(const std::string &piece) const = 0;
+
+  /**
+   * Returns the size of the vocabulary.
+   *
+   * @return The number of tokens in the vocabulary.
+   */
+  virtual int32_t vocab_size() const = 0;
+
+  /**
+   * Returns whether the token is a special token.
+   *
+   * @param token The token ID.
+   * @return True if the token is a special token, false otherwise.
+   */
+  virtual bool is_special_token(uint64_t token) const = 0;
+
+  /**
+   * Returns whether the model is loaded.
+   *
+   * @return True if the model is loaded, false otherwise.
+   */
+  virtual bool is_loaded() const = 0;
+
+  /**
+   * Helper to split input text into a special token and the preceding regular
+   * text.
+   *
+   * @param input The input string.
+   * @param offset The starting offset.
+   * @return A pair of (matched special token string, preceding regular text).
+   */
+  virtual std::pair<std::optional<std::string>, std::string>
+  split_with_allowed_special_token(const std::string &input,
+                                   size_t offset) const = 0;
+
+  virtual uint64_t bos_token_id() const = 0;
+  virtual uint64_t eos_token_id() const = 0;
+};
+
+// -- Factory ------------------------------------------------------------------
+
+// Helper macro to standardize addition of config member fields
+#define MODEL_CONFIG_MEMBER(type, name)                                        \
+  std::optional<type> name;                                                    \
+  ModelConfig &set_##name(type arg) {                                          \
+    this->name = std::move(arg);                                               \
+    return *this;                                                              \
+  }
+
+/**
+ * Factory and config class for creating a new Model
+ */
+class ModelConfig {
+public:
+  std::string type;
+
+  // Data for BPEModel
+  using TokenPairs = std::vector<std::pair<std::string, uint64_t>>;
+  MODEL_CONFIG_MEMBER(TokenPairs, token_pairs)
+  MODEL_CONFIG_MEMBER(TokenPairs, special_token_pairs)
+
+  MODEL_CONFIG_MEMBER(std::vector<std::string>, merges)
+  MODEL_CONFIG_MEMBER(bool, byte_fallback)
+  MODEL_CONFIG_MEMBER(std::string, unk_token)
+  MODEL_CONFIG_MEMBER(std::string, bos_token)
+  MODEL_CONFIG_MEMBER(std::string, eos_token)
+  MODEL_CONFIG_MEMBER(std::string, continuing_subword_prefix)
+  MODEL_CONFIG_MEMBER(size_t, max_input_chars_per_word)
+
+  // Paths for extra config files (HuggingFace specific)
+  MODEL_CONFIG_MEMBER(std::string, model_config_path)
+  MODEL_CONFIG_MEMBER(std::string, special_tokens_map_path)
+
+  ModelConfig() = default;
+
+  /**
+   * Populate from a json config file (the root tokenizer.json)
+   */
+  ModelConfig &parse_json(const nlohmann::json &json_config);
+
+  /**
+   * Construct the model instance from the member data
+   */
+  Model::Ptr create() const;
+};
+
+} // namespace tokenizers

package/third-party/include/executorch/extension/llm/tokenizers/include/pytorch/tokenizers/normalizer.h

@@ -101,13 +101,22 @@ public:
   /**
    * Used by: SequenceNormalizer
    */
-  NORMALIZER_CONFIG_MEMBER(std::vector<NormalizerConfig>, normalizers)
+  using Configs = std::vector<NormalizerConfig>;
+  NORMALIZER_CONFIG_MEMBER(Configs, normalizers)
 
   /**
    * Used by: PrependNormalizer
    */
   NORMALIZER_CONFIG_MEMBER(std::string, prepend)
 
+  /**
+   * Used by: BertNormalizer
+   */
+  NORMALIZER_CONFIG_MEMBER(bool, clean_text)
+  NORMALIZER_CONFIG_MEMBER(bool, handle_chinese_chars)
+  NORMALIZER_CONFIG_MEMBER(bool, lowercase)
+  NORMALIZER_CONFIG_MEMBER(bool, strip_accents)
+
   /*----------------*/
   /* Public methods */
   /*----------------*/
@@ -210,4 +219,49 @@ public:
 
 }; // end class NFCNormalizer
 
+// -- Lowercase ----------------------------------------------------------------
+// Used for lowercasing the input
+// CITE:
+// https://github.com/huggingface/tokenizers/blob/main/tokenizers/src/normalizers/utils.rs
+
+class LowercaseNormalizer : public Normalizer {
+public:
+  /** Default constructor */
+  explicit LowercaseNormalizer() = default;
+
+  /** Lowercase the input */
+  std::string normalize(const std::string &input) const override;
+
+}; // end class LowercaseNormalizer
+
+// -- Bert ---------------------------------------------------------------------
+// Used for BERT-style normalization (cleaning, lowercasing, accent removal)
+// CITE:
+// https://github.com/huggingface/tokenizers/blob/main/tokenizers/src/normalizers/bert.rs
+
+class BertNormalizer : public Normalizer {
+public:
+  /**
+   * @param clean_text: Whether to clean the text (remove control chars, etc.)
+   * @param handle_chinese_chars: Whether to put spaces around Chinese
+   * characters
+   * @param lowercase: Whether to lowercase the input
+   * @param strip_accents: Whether to strip accents (optional, usually follows
+   * lowercase)
+   */
+  explicit BertNormalizer(bool clean_text, bool handle_chinese_chars,
+                          bool lowercase, std::optional<bool> strip_accents)
+      : clean_text_(clean_text), handle_chinese_chars_(handle_chinese_chars),
+        lowercase_(lowercase), strip_accents_(strip_accents) {}
+
+  /** Perform BERT normalization steps */
+  std::string normalize(const std::string &input) const override;
+
+protected:
+  const bool clean_text_;
+  const bool handle_chinese_chars_;
+  const bool lowercase_;
+  const std::optional<bool> strip_accents_;
+};
+
 } // namespace tokenizers

package/third-party/include/executorch/extension/llm/tokenizers/include/pytorch/tokenizers/padding.h

@@ -0,0 +1,112 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+// @lint-ignore-every LICENSELINT
+
+#pragma once
+
+// Standard
+#include <memory>
+#include <optional>
+#include <string>
+#include <vector>
+
+// Third Party
+#include <nlohmann/json.hpp>
+
+namespace tokenizers {
+
+// -- Padding ------------------------------------------------------------------
+
+enum class PaddingDirection {
+  Left,
+  Right,
+};
+
+enum class PaddingStrategy {
+  BatchLongest,
+  Fixed,
+};
+
+struct PaddingParams {
+  PaddingStrategy strategy = PaddingStrategy::BatchLongest;
+  PaddingDirection direction = PaddingDirection::Right;
+  std::optional<size_t> fixed_size;
+  std::optional<size_t> pad_to_multiple_of;
+  uint32_t pad_id = 0;
+  uint32_t pad_type_id = 0;
+  std::string pad_token = "[PAD]";
+};
+
+class Padding {
+public:
+  /** Shared pointer type */
+  typedef std::shared_ptr<Padding> Ptr;
+
+  /**
+   * @param params: The padding parameters
+   */
+  explicit Padding(const PaddingParams &params);
+
+  /**
+   * Pad the tokens according to the configuration
+   */
+  std::vector<uint64_t> pad(std::vector<uint64_t> tokens) const;
+
+  /**
+   * Generate attention mask for the padded tokens.
+   * 1 for real tokens, 0 for padded tokens.
+   */
+  std::vector<uint32_t> generate_mask(const std::vector<uint64_t> &tokens,
+                                      size_t padded_size) const;
+
+private:
+  PaddingParams params_;
+};
+
+// -- Factory ------------------------------------------------------------------
+
+// Helper macro to standardize addition of config member fields
+#define PADDING_CONFIG_MEMBER(type, name)                                      \
+  PaddingConfig &set_##name(type arg) {                                        \
+    this->params.name = std::move(arg);                                        \
+    return *this;                                                              \
+  }
+
+class PaddingConfig {
+public:
+  explicit PaddingConfig(std::string strategy = "");
+
+  /**
+   * Construct the padding instance from the member data
+   */
+  Padding::Ptr create() const;
+
+  /**
+   * Populate from a json config file
+   */
+  PaddingConfig &parse_json(const nlohmann::json &json_config);
+
+  // Configuration members
+  PaddingParams params;
+
+  PADDING_CONFIG_MEMBER(PaddingStrategy, strategy)
+  PADDING_CONFIG_MEMBER(PaddingDirection, direction)
+
+  PaddingConfig &set_fixed_size(std::optional<size_t> arg) {
+    this->params.fixed_size = std::move(arg);
+    this->params.strategy = PaddingStrategy::Fixed;
+    return *this;
+  }
+
+  PADDING_CONFIG_MEMBER(std::optional<size_t>, pad_to_multiple_of)
+  PADDING_CONFIG_MEMBER(uint32_t, pad_id)
+  PADDING_CONFIG_MEMBER(uint32_t, pad_type_id)
+  PADDING_CONFIG_MEMBER(std::string, pad_token)
+};
+
+} // namespace tokenizers