react-native-executorch 0.7.0 → 0.7.1

package/third-party/include/executorch/extension/llm/tokenizers/include/pytorch/tokenizers/model.h

@@ -0,0 +1,151 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+// @lint-ignore-every LICENSELINT
+
+#pragma once
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include <nlohmann/json.hpp>
+#include <pytorch/tokenizers/map_utils.h>
+#include <pytorch/tokenizers/result.h>
+#include <pytorch/tokenizers/string_integer_map.h>
+
+namespace tokenizers {
+
+// -- Base ---------------------------------------------------------------------
+
+/**
+ * Abstract base class for tokenization models.
+ *
+ * A Model corresponds to the core logic that converts a piece of text (usually
+ * resulting from the pre-tokenization step) into a sequence of token IDs, and
+ * vice-versa.
+ *
+ * It encapsulates the vocabulary and the algorithm (e.g., BPE, WordPiece,
+ * Unigram).
+ */
+class Model {
+public:
+  using Ptr = std::shared_ptr<Model>;
+
+  virtual ~Model() = default;
+
+  /**
+   * Tokenizes a string piece into a sequence of token IDs.
+   *
+   * @param piece The input string to tokenize.
+   * @return A Result containing the vector of token IDs.
+   */
+  virtual Result<std::vector<uint64_t>>
+  tokenize(const std::string &piece) const = 0;
+
+  /**
+   * Converts a token ID to its string representation.
+   *
+   * @param token The token ID.
+   * @return A Result containing the string representation of the token.
+   */
+  virtual Result<std::string> id_to_piece(uint64_t token) const = 0;
+
+  /**
+   * Converts a string representation to its token ID.
+   *
+   * @param piece The string representation of the token.
+   * @return A Result containing the token ID.
+   */
+  virtual Result<uint64_t> piece_to_id(const std::string &piece) const = 0;
+
+  /**
+   * Returns the size of the vocabulary.
+   *
+   * @return The number of tokens in the vocabulary.
+   */
+  virtual int32_t vocab_size() const = 0;
+
+  /**
+   * Returns whether the token is a special token.
+   *
+   * @param token The token ID.
+   * @return True if the token is a special token, false otherwise.
+   */
+  virtual bool is_special_token(uint64_t token) const = 0;
+
+  /**
+   * Returns whether the model is loaded.
+   *
+   * @return True if the model is loaded, false otherwise.
+   */
+  virtual bool is_loaded() const = 0;
+
+  /**
+   * Helper to split input text into a special token and the preceding regular
+   * text.
+   *
+   * @param input The input string.
+   * @param offset The starting offset.
+   * @return A pair of (matched special token string, preceding regular text).
+   */
+  virtual std::pair<std::optional<std::string>, std::string>
+  split_with_allowed_special_token(const std::string &input,
+                                   size_t offset) const = 0;
+
+  virtual uint64_t bos_token_id() const = 0;
+  virtual uint64_t eos_token_id() const = 0;
+};
+
+// -- Factory ------------------------------------------------------------------
+
+// Helper macro to standardize addition of config member fields
+#define MODEL_CONFIG_MEMBER(type, name)                                        \
+  std::optional<type> name;                                                    \
+  ModelConfig &set_##name(type arg) {                                          \
+    this->name = std::move(arg);                                               \
+    return *this;                                                              \
+  }
+
+/**
+ * Factory and config class for creating a new Model
+ */
+class ModelConfig {
+public:
+  std::string type;
+
+  // Data for BPEModel
+  using TokenPairs = std::vector<std::pair<std::string, uint64_t>>;
+  MODEL_CONFIG_MEMBER(TokenPairs, token_pairs)
+  MODEL_CONFIG_MEMBER(TokenPairs, special_token_pairs)
+
+  MODEL_CONFIG_MEMBER(std::vector<std::string>, merges)
+  MODEL_CONFIG_MEMBER(bool, byte_fallback)
+  MODEL_CONFIG_MEMBER(std::string, unk_token)
+  MODEL_CONFIG_MEMBER(std::string, bos_token)
+  MODEL_CONFIG_MEMBER(std::string, eos_token)
+  MODEL_CONFIG_MEMBER(std::string, continuing_subword_prefix)
+  MODEL_CONFIG_MEMBER(size_t, max_input_chars_per_word)
+
+  // Paths for extra config files (HuggingFace specific)
+  MODEL_CONFIG_MEMBER(std::string, model_config_path)
+  MODEL_CONFIG_MEMBER(std::string, special_tokens_map_path)
+
+  ModelConfig() = default;
+
+  /**
+   * Populate from a json config file (the root tokenizer.json)
+   */
+  ModelConfig &parse_json(const nlohmann::json &json_config);
+
+  /**
+   * Construct the model instance from the member data
+   */
+  Model::Ptr create() const;
+};
+
+} // namespace tokenizers

package/third-party/include/executorch/extension/llm/tokenizers/include/pytorch/tokenizers/normalizer.h

@@ -101,13 +101,22 @@ public:
   /**
    * Used by: SequenceNormalizer
    */
-  NORMALIZER_CONFIG_MEMBER(std::vector<NormalizerConfig>, normalizers)
+  using Configs = std::vector<NormalizerConfig>;
+  NORMALIZER_CONFIG_MEMBER(Configs, normalizers)
 
   /**
    * Used by: PrependNormalizer
    */
   NORMALIZER_CONFIG_MEMBER(std::string, prepend)
 
+  /**
+   * Used by: BertNormalizer
+   */
+  NORMALIZER_CONFIG_MEMBER(bool, clean_text)
+  NORMALIZER_CONFIG_MEMBER(bool, handle_chinese_chars)
+  NORMALIZER_CONFIG_MEMBER(bool, lowercase)
+  NORMALIZER_CONFIG_MEMBER(bool, strip_accents)
+
   /*----------------*/
   /* Public methods */
   /*----------------*/
@@ -210,4 +219,49 @@ public:
 
 }; // end class NFCNormalizer
 
+// -- Lowercase ----------------------------------------------------------------
+// Used for lowercasing the input
+// CITE:
+// https://github.com/huggingface/tokenizers/blob/main/tokenizers/src/normalizers/utils.rs
+
+class LowercaseNormalizer : public Normalizer {
+public:
+  /** Default constructor */
+  explicit LowercaseNormalizer() = default;
+
+  /** Lowercase the input */
+  std::string normalize(const std::string &input) const override;
+
+}; // end class LowercaseNormalizer
+
+// -- Bert ---------------------------------------------------------------------
+// Used for BERT-style normalization (cleaning, lowercasing, accent removal)
+// CITE:
+// https://github.com/huggingface/tokenizers/blob/main/tokenizers/src/normalizers/bert.rs
+
+class BertNormalizer : public Normalizer {
+public:
+  /**
+   * @param clean_text: Whether to clean the text (remove control chars, etc.)
+   * @param handle_chinese_chars: Whether to put spaces around Chinese
+   * characters
+   * @param lowercase: Whether to lowercase the input
+   * @param strip_accents: Whether to strip accents (optional, usually follows
+   * lowercase)
+   */
+  explicit BertNormalizer(bool clean_text, bool handle_chinese_chars,
+                          bool lowercase, std::optional<bool> strip_accents)
+      : clean_text_(clean_text), handle_chinese_chars_(handle_chinese_chars),
+        lowercase_(lowercase), strip_accents_(strip_accents) {}
+
+  /** Perform BERT normalization steps */
+  std::string normalize(const std::string &input) const override;
+
+protected:
+  const bool clean_text_;
+  const bool handle_chinese_chars_;
+  const bool lowercase_;
+  const std::optional<bool> strip_accents_;
+};
+
 } // namespace tokenizers

package/third-party/include/executorch/extension/llm/tokenizers/include/pytorch/tokenizers/padding.h

@@ -0,0 +1,112 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+// @lint-ignore-every LICENSELINT
+
+#pragma once
+
+// Standard
+#include <memory>
+#include <optional>
+#include <string>
+#include <vector>
+
+// Third Party
+#include <nlohmann/json.hpp>
+
+namespace tokenizers {
+
+// -- Padding ------------------------------------------------------------------
+
+enum class PaddingDirection {
+  Left,
+  Right,
+};
+
+enum class PaddingStrategy {
+  BatchLongest,
+  Fixed,
+};
+
+struct PaddingParams {
+  PaddingStrategy strategy = PaddingStrategy::BatchLongest;
+  PaddingDirection direction = PaddingDirection::Right;
+  std::optional<size_t> fixed_size;
+  std::optional<size_t> pad_to_multiple_of;
+  uint32_t pad_id = 0;
+  uint32_t pad_type_id = 0;
+  std::string pad_token = "[PAD]";
+};
+
+class Padding {
+public:
+  /** Shared pointer type */
+  typedef std::shared_ptr<Padding> Ptr;
+
+  /**
+   * @param params: The padding parameters
+   */
+  explicit Padding(const PaddingParams &params);
+
+  /**
+   * Pad the tokens according to the configuration
+   */
+  std::vector<uint64_t> pad(std::vector<uint64_t> tokens) const;
+
+  /**
+   * Generate attention mask for the padded tokens.
+   * 1 for real tokens, 0 for padded tokens.
+   */
+  std::vector<uint32_t> generate_mask(const std::vector<uint64_t> &tokens,
+                                      size_t padded_size) const;
+
+private:
+  PaddingParams params_;
+};
+
+// -- Factory ------------------------------------------------------------------
+
+// Helper macro to standardize addition of config member fields
+#define PADDING_CONFIG_MEMBER(type, name)                                      \
+  PaddingConfig &set_##name(type arg) {                                        \
+    this->params.name = std::move(arg);                                        \
+    return *this;                                                              \
+  }
+
+class PaddingConfig {
+public:
+  explicit PaddingConfig(std::string strategy = "");
+
+  /**
+   * Construct the padding instance from the member data
+   */
+  Padding::Ptr create() const;
+
+  /**
+   * Populate from a json config file
+   */
+  PaddingConfig &parse_json(const nlohmann::json &json_config);
+
+  // Configuration members
+  PaddingParams params;
+
+  PADDING_CONFIG_MEMBER(PaddingStrategy, strategy)
+  PADDING_CONFIG_MEMBER(PaddingDirection, direction)
+
+  PaddingConfig &set_fixed_size(std::optional<size_t> arg) {
+    this->params.fixed_size = std::move(arg);
+    this->params.strategy = PaddingStrategy::Fixed;
+    return *this;
+  }
+
+  PADDING_CONFIG_MEMBER(std::optional<size_t>, pad_to_multiple_of)
+  PADDING_CONFIG_MEMBER(uint32_t, pad_id)
+  PADDING_CONFIG_MEMBER(uint32_t, pad_type_id)
+  PADDING_CONFIG_MEMBER(std::string, pad_token)
+};
+
+} // namespace tokenizers

package/third-party/include/executorch/extension/llm/tokenizers/include/pytorch/tokenizers/post_processor.h

@@ -38,6 +38,10 @@ public:
 
   /**
    * Process the token IDs (single sequence).
+   *
+   * NOTE: Unlike the Rust implementation which  uses a single method
+   * taking  Encoding and an Option<Encoding>, we use overloads here
+   * to explicitly handle single vs pair sequences while processing raw IDs.
    */
   virtual std::vector<uint64_t>
   process(const std::vector<uint64_t> &tokens,
@@ -54,27 +58,65 @@ public:
 
 // -- Factory/Common Types -----------------------------------------------------
 
+// Helper macro to standardize addition of config member fields
+#define POST_PROCESSOR_CONFIG_MEMBER(type, name)                               \
+  std::optional<type> name;                                                    \
+  PostProcessorConfig &set_##name(type arg) {                                  \
+    this->name = std::move(arg);                                               \
+    return *this;                                                              \
+  }
+
 enum class SequenceId { A, B };
 
+struct SpecialToken {
+  std::string id;
+  std::vector<uint64_t> ids;
+  std::vector<std::string> tokens;
+};
+
 struct Piece {
   bool is_special_token;
   std::string id; // For SpecialToken (e.g. "[CLS]"). For Sequence (e.g. "A").
-  uint32_t type_id;
+  uint64_t type_id;
 
-  static Piece Sequence(SequenceId id, uint32_t type_id) {
+  static Piece Sequence(SequenceId id, uint64_t type_id) {
     return {false, id == SequenceId::A ? "A" : "B", type_id};
   }
-  static Piece SpecialToken(std::string id, uint32_t type_id) {
+  static Piece SpecialToken(std::string id, uint64_t type_id) {
     return {true, std::move(id), type_id};
   }
 };
 
 using Template = std::vector<Piece>;
+// -- Config -------------------------------------------------------------------
 
-struct SpecialToken {
-  std::string id;
-  std::vector<uint32_t> ids;
-  std::vector<std::string> tokens;
+class PostProcessorConfig {
+public:
+  using SpecialTokenMap = std::map<std::string, tokenizers::SpecialToken>;
+  using StringIdPair = std::pair<std::string, uint64_t>;
+
+  std::string type;
+
+  // TemplateProcessing
+  POST_PROCESSOR_CONFIG_MEMBER(Template, single)
+  POST_PROCESSOR_CONFIG_MEMBER(Template, pair)
+  POST_PROCESSOR_CONFIG_MEMBER(SpecialTokenMap, special_tokens)
+
+  // Bert / Roberta (unused params in no-op, but kept for parsing logic)
+  POST_PROCESSOR_CONFIG_MEMBER(StringIdPair, sep)
+  POST_PROCESSOR_CONFIG_MEMBER(StringIdPair, cls)
+  POST_PROCESSOR_CONFIG_MEMBER(bool, trim_offsets)
+  POST_PROCESSOR_CONFIG_MEMBER(bool, add_prefix_space)
+
+  // Sequence
+  using Configs = std::vector<PostProcessorConfig>;
+  POST_PROCESSOR_CONFIG_MEMBER(Configs, processors)
+
+  explicit PostProcessorConfig(std::string type = "");
+
+  PostProcessor::Ptr create() const;
+
+  PostProcessorConfig &parse_json(const nlohmann::json &json_config);
 };
 
 // -- TemplateProcessing -------------------------------------------------------
@@ -106,11 +148,9 @@ private:
                                        bool add_special_tokens) const;
 };
 
-// -- BertProcessing -----------------------------------------------------------
-
-class BertProcessing : public PostProcessor {
+class Sequence : public PostProcessor {
 public:
-  BertProcessing();
+  explicit Sequence(std::vector<PostProcessor::Ptr> processors);
 
   size_t added_tokens(bool is_pair) const override;
 
@@ -120,13 +160,17 @@ public:
   std::vector<uint64_t> process(const std::vector<uint64_t> &tokens_a,
                                 const std::vector<uint64_t> &tokens_b,
                                 bool add_special_tokens = true) const override;
-};
 
-// -- RobertaProcessing --------------------------------------------------------
+private:
+  std::vector<PostProcessor::Ptr> processors_;
+};
 
-class RobertaProcessing : public PostProcessor {
+// -- BertProcessing -----------------------------------------------------------
+// Used for BERT post-processing (adding special tokens)
+class BertProcessing : public PostProcessor {
 public:
-  RobertaProcessing();
+  BertProcessing(std::pair<std::string, uint64_t> sep,
+                 std::pair<std::string, uint64_t> cls);
 
   size_t added_tokens(bool is_pair) const override;
 
@@ -136,13 +180,19 @@ public:
   std::vector<uint64_t> process(const std::vector<uint64_t> &tokens_a,
                                 const std::vector<uint64_t> &tokens_b,
                                 bool add_special_tokens = true) const override;
-};
 
-// -- Sequence -----------------------------------------------------------------
+private:
+  std::pair<std::string, uint64_t> sep_;
+  std::pair<std::string, uint64_t> cls_;
+};
 
-class Sequence : public PostProcessor {
+// -- RobertaProcessing --------------------------------------------------------
+// Used for RoBERTa post-processing
+class RobertaProcessing : public PostProcessor {
 public:
-  explicit Sequence(std::vector<PostProcessor::Ptr> processors);
+  RobertaProcessing(std::pair<std::string, uint64_t> sep,
+                    std::pair<std::string, uint64_t> cls, bool trim_offsets,
+                    bool add_prefix_space);
 
   size_t added_tokens(bool is_pair) const override;
 
@@ -154,34 +204,43 @@ public:
                                 bool add_special_tokens = true) const override;
 
 private:
-  std::vector<PostProcessor::Ptr> processors_;
+  std::pair<std::string, uint64_t> sep_;
+  std::pair<std::string, uint64_t> cls_;
+  bool trim_offsets_;
+  bool add_prefix_space_;
 };
 
-// -- Config -------------------------------------------------------------------
-
-class PostProcessorConfig {
-public:
-  std::string type;
-
-  // TemplateProcessing
-  Template single;
-  Template pair;
-  std::map<std::string, SpecialToken> special_tokens;
-
-  // Bert / Roberta (unused params in no-op, but kept for parsing logic)
-  std::pair<std::string, uint32_t> sep;
-  std::pair<std::string, uint32_t> cls;
-  bool trim_offsets = true;
-  bool add_prefix_space = true;
+// -- ByteLevel
+// ----------------------------------------------------------------
+// TODO: Implement ByteLevelProcessor
+// This is a broader issue, as most of the processing is done on offsets.
+// Our current implementation doesn't supoort it and would require us to
+// introduce a complex Encoding type. Something similiar to the originl hf
+// implementaiton:
+// https://github.com/huggingface/tokenizers/blob/main/tokenizers/src/tokenizer/encoding.rs
+// so we could store the offsets from pretokenization step.
+/*
+class ByteLevel : public PostProcessor {
+ public:
+  ByteLevel(bool trim_offsets, bool add_prefix_space);
 
-  // Sequence
-  std::vector<PostProcessorConfig> processors;
+  size_t added_tokens(bool is_pair) const override;
 
-  explicit PostProcessorConfig(std::string type = "");
+  std::vector<uint64_t> process(
+      const std::vector<uint64_t>& tokens,
+      bool add_special_tokens = true) const override;
 
-  PostProcessor::Ptr create() const;
+  std::vector<uint64_t> process(
+      const std::vector<uint64_t>& tokens_a,
+      const std::vector<uint64_t>& tokens_b,
+      bool add_special_tokens = true) const override;
 
-  PostProcessorConfig &parse_json(const nlohmann::json &json_config);
+ private:
+  bool trim_offsets_;
+  bool add_prefix_space_;
 };
+*/
 
-} // namespace tokenizers
+// -- Sequence
+// -----------------------------------------------------------------
+} // namespace tokenizers

package/third-party/include/executorch/extension/llm/tokenizers/include/pytorch/tokenizers/pre_tokenizer.h

@@ -53,7 +53,7 @@ public:
 // -- Factory ------------------------------------------------------------------
 
 // Helper macro to standardize addition of config member fields
-#define CONFIG_MEMBER(type, name)                                              \
+#define PRETOKENIZER_CONFIG_MEMBER(type, name)                                 \
   std::optional<type> name;                                                    \
   PreTokenizerConfig &set_##name(type arg) {                                   \
     this->name = std::move(arg);                                               \
@@ -92,37 +92,38 @@ public:
   /**
    * Used by: RegexPreTokenizer, ByteLevelPreTokenizer
    */
-  CONFIG_MEMBER(std::string, pattern)
+  PRETOKENIZER_CONFIG_MEMBER(std::string, pattern)
 
   /**
    * Used by: DigitsPreTokenizer
    */
-  CONFIG_MEMBER(bool, individual_digits)
+  PRETOKENIZER_CONFIG_MEMBER(bool, individual_digits)
 
   /**
    * Used by: ByteLevelPreTokenizer
    */
-  CONFIG_MEMBER(bool, add_prefix_space)
+  PRETOKENIZER_CONFIG_MEMBER(bool, add_prefix_space)
 
   /**
    * Used by RegexPreTokenizer
    */
-  CONFIG_MEMBER(bool, is_delimiter)
+  PRETOKENIZER_CONFIG_MEMBER(bool, is_delimiter)
 
   /**
    * Used by RegexPreTokenizer - Split behavior
    */
-  CONFIG_MEMBER(std::string, behavior)
+  PRETOKENIZER_CONFIG_MEMBER(std::string, behavior)
 
   /**
    * Used by RegexPreTokenizer - Split invert flag
    */
-  CONFIG_MEMBER(bool, invert)
+  PRETOKENIZER_CONFIG_MEMBER(bool, invert)
 
   /**
    * Used by: SequencePreTokenizer
    */
-  CONFIG_MEMBER(std::vector<PreTokenizerConfig>, pretokenizers)
+  using Configs = std::vector<PreTokenizerConfig>;
+  PRETOKENIZER_CONFIG_MEMBER(Configs, pretokenizers)
 
   /*----------------*/
   /* Public methods */
@@ -259,6 +260,21 @@ public:
 private:
   const std::vector<PreTokenizer::Ptr> pre_tokenizers_;
 
-}; // end class ByteLevelPreTokenizer
+}; // end class SequencePreTokenizer
+
+// -- Bert ---------------------------------------------------------------------
+// Used for BERT-style pre-tokenization (splitting on whitespace and
+// punctuation) CITE:
+// https://github.com/huggingface/tokenizers/blob/main/tokenizers/src/pre_tokenizers/bert.rs
+
+class BertPreTokenizer : public PreTokenizer {
+public:
+  BertPreTokenizer() = default;
+
+  /** Perform BERT pre-tokenization */
+  std::vector<std::string>
+  pre_tokenize(const std::string &input) const override;
+
+}; // end class BertPreTokenizer
 
 } // namespace tokenizers

package/third-party/include/executorch/extension/llm/tokenizers/include/pytorch/tokenizers/token_decoder.h

@@ -55,6 +55,14 @@ public:
 
 // -- Factory ------------------------------------------------------------------
 
+// Helper macro to standardize addition of config member fields
+#define TOKEN_DECODER_CONFIG_MEMBER(type, name)                                \
+  std::optional<type> name;                                                    \
+  TokenDecoderConfig &set_##name(type arg) {                                   \
+    this->name = std::move(arg);                                               \
+    return *this;                                                              \
+  }
+
 /**
  * Factory and config class for creating a new TokenDecoder
  */
@@ -67,16 +75,20 @@ public:
   std::string type;
 
   // Parameters for Replace decoder
-  std::string replace_pattern;
-  std::string replace_content;
+  TOKEN_DECODER_CONFIG_MEMBER(std::string, replace_pattern)
+  TOKEN_DECODER_CONFIG_MEMBER(std::string, replace_content)
 
   // Parameters for Sequence decoder
-  std::vector<nlohmann::json> sequence_decoders;
+  TOKEN_DECODER_CONFIG_MEMBER(std::vector<nlohmann::json>, sequence_decoders)
 
   // Parameters for Strip decoder
-  std::string strip_content;
-  size_t strip_start;
-  size_t strip_stop;
+  TOKEN_DECODER_CONFIG_MEMBER(std::string, strip_content)
+  TOKEN_DECODER_CONFIG_MEMBER(size_t, strip_start)
+  TOKEN_DECODER_CONFIG_MEMBER(size_t, strip_stop)
+
+  // Parameters for WordPiece decoder
+  TOKEN_DECODER_CONFIG_MEMBER(std::string, wordpiece_prefix)
+  TOKEN_DECODER_CONFIG_MEMBER(bool, wordpiece_cleanup)
 
   /*----------------*/
   /* Public methods */
@@ -161,6 +173,21 @@ private:
   size_t stop_;
 }; // end class StripTokenDecoder
 
+// -- WordPiece ----------------------------------------------------------------
+// Used for WordPiece decoding
+
+class WordPieceTokenDecoder : public TokenDecoder {
+public:
+  explicit WordPieceTokenDecoder(std::string prefix = "##",
+                                 bool cleanup = true);
+  std::vector<std::string>
+  decode(const std::vector<std::string> &tokens) const override;
+
+private:
+  std::string prefix_;
+  bool cleanup_;
+}; // end class WordPieceTokenDecoder
+
 // -- Sequence -----------------------------------------------------------------
 // Applies a sequence of decoders in order