tokenizers 0.23.2__tar.gz → 0.23.3__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (187) hide show
  1. {tokenizers-0.23.2 → tokenizers-0.23.3}/PKG-INFO +2 -2
  2. {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/Cargo.lock +1 -1
  3. {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/Cargo.toml +1 -1
  4. {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/Makefile +3 -1
  5. {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/src/models.rs +2 -13
  6. {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/src/trainers.rs +1 -1
  7. {tokenizers-0.23.2 → tokenizers-0.23.3}/py_src/tokenizers/__init__.pyi +7 -7
  8. tokenizers-0.23.2/py_src/tokenizers/decoders.pyi → tokenizers-0.23.3/py_src/tokenizers/decoders/__init__.pyi +1 -1
  9. tokenizers-0.23.2/py_src/tokenizers/normalizers.pyi → tokenizers-0.23.3/py_src/tokenizers/normalizers/__init__.pyi +8 -8
  10. tokenizers-0.23.2/py_src/tokenizers/pre_tokenizers.pyi → tokenizers-0.23.3/py_src/tokenizers/pre_tokenizers/__init__.pyi +4 -4
  11. tokenizers-0.23.2/py_src/tokenizers/processors.pyi → tokenizers-0.23.3/py_src/tokenizers/processors/__init__.pyi +2 -2
  12. tokenizers-0.23.2/py_src/tokenizers/trainers.pyi → tokenizers-0.23.3/py_src/tokenizers/trainers/__init__.pyi +161 -14
  13. {tokenizers-0.23.2 → tokenizers-0.23.3}/pyproject.toml +1 -1
  14. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/tokenizer/mod.rs +60 -34
  15. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/tokenizer/pre_tokenizer.rs +0 -62
  16. tokenizers-0.23.2/py_src/tokenizers/trainers/__init__.pyi +0 -712
  17. {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/.cargo/config.toml +0 -0
  18. {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/.gitignore +0 -0
  19. {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/CHANGELOG.md +0 -0
  20. {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/MANIFEST.in +0 -0
  21. {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/README.md +0 -0
  22. {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/benches/test_tiktoken.py +0 -0
  23. {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/conftest.py +0 -0
  24. {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/docs/pyo3.md +0 -0
  25. {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/examples/train_parity_bpe.py +0 -0
  26. {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/pytest.ini +0 -0
  27. {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/rust-toolchain +0 -0
  28. {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/scripts/convert.py +0 -0
  29. {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/scripts/sentencepiece_extractor.py +0 -0
  30. {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/scripts/spm_parity_check.py +0 -0
  31. {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/setup.cfg +0 -0
  32. {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/src/decoders.rs +0 -0
  33. {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/src/encoding.rs +0 -0
  34. {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/src/error.rs +0 -0
  35. {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/src/lib.rs +0 -0
  36. {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/src/normalizers.rs +0 -0
  37. {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/src/pre_tokenizers.rs +0 -0
  38. {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/src/processors.rs +0 -0
  39. {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/src/token.rs +0 -0
  40. {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/src/tokenizer.rs +0 -0
  41. {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/src/utils/iterators.rs +0 -0
  42. {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/src/utils/mod.rs +0 -0
  43. {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/src/utils/normalization.rs +0 -0
  44. {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/src/utils/pretokenization.rs +0 -0
  45. {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/src/utils/regex.rs +0 -0
  46. {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/src/utils/serde_pyo3.rs +0 -0
  47. {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/test.txt +0 -0
  48. {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/tests/__init__.py +0 -0
  49. {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/tests/bindings/__init__.py +0 -0
  50. {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/tests/bindings/test_decoders.py +0 -0
  51. {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/tests/bindings/test_encoding.py +0 -0
  52. {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/tests/bindings/test_models.py +0 -0
  53. {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/tests/bindings/test_normalizers.py +0 -0
  54. {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/tests/bindings/test_pre_tokenizers.py +0 -0
  55. {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/tests/bindings/test_processors.py +0 -0
  56. {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/tests/bindings/test_tokenizer.py +0 -0
  57. {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/tests/bindings/test_trainers.py +0 -0
  58. {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/tests/documentation/__init__.py +0 -0
  59. {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/tests/documentation/test_pipeline.py +0 -0
  60. {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/tests/documentation/test_quicktour.py +0 -0
  61. {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/tests/documentation/test_tutorial_train_from_iterators.py +0 -0
  62. {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/tests/implementations/__init__.py +0 -0
  63. {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/tests/implementations/test_base_tokenizer.py +0 -0
  64. {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/tests/implementations/test_bert_wordpiece.py +0 -0
  65. {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/tests/implementations/test_byte_level_bpe.py +0 -0
  66. {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/tests/implementations/test_char_bpe.py +0 -0
  67. {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/tests/implementations/test_sentencepiece.py +0 -0
  68. {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/tests/test_benchmarks.py +0 -0
  69. {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/tests/test_freethreaded.py +0 -0
  70. {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/tests/test_serialization.py +0 -0
  71. {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/tests/utils.py +0 -0
  72. {tokenizers-0.23.2 → tokenizers-0.23.3}/py_src/tokenizers/__init__.py +0 -0
  73. {tokenizers-0.23.2 → tokenizers-0.23.3}/py_src/tokenizers/decoders/__init__.py +0 -0
  74. {tokenizers-0.23.2 → tokenizers-0.23.3}/py_src/tokenizers/implementations/__init__.py +0 -0
  75. {tokenizers-0.23.2 → tokenizers-0.23.3}/py_src/tokenizers/implementations/base_tokenizer.py +0 -0
  76. {tokenizers-0.23.2 → tokenizers-0.23.3}/py_src/tokenizers/implementations/bert_wordpiece.py +0 -0
  77. {tokenizers-0.23.2 → tokenizers-0.23.3}/py_src/tokenizers/implementations/byte_level_bpe.py +0 -0
  78. {tokenizers-0.23.2 → tokenizers-0.23.3}/py_src/tokenizers/implementations/char_level_bpe.py +0 -0
  79. {tokenizers-0.23.2 → tokenizers-0.23.3}/py_src/tokenizers/implementations/sentencepiece_bpe.py +0 -0
  80. {tokenizers-0.23.2 → tokenizers-0.23.3}/py_src/tokenizers/implementations/sentencepiece_unigram.py +0 -0
  81. {tokenizers-0.23.2 → tokenizers-0.23.3}/py_src/tokenizers/models/__init__.py +0 -0
  82. /tokenizers-0.23.2/py_src/tokenizers/models.pyi → /tokenizers-0.23.3/py_src/tokenizers/models/__init__.pyi +0 -0
  83. {tokenizers-0.23.2 → tokenizers-0.23.3}/py_src/tokenizers/normalizers/__init__.py +0 -0
  84. {tokenizers-0.23.2 → tokenizers-0.23.3}/py_src/tokenizers/pre_tokenizers/__init__.py +0 -0
  85. {tokenizers-0.23.2 → tokenizers-0.23.3}/py_src/tokenizers/processors/__init__.py +0 -0
  86. {tokenizers-0.23.2 → tokenizers-0.23.3}/py_src/tokenizers/py.typed +0 -0
  87. {tokenizers-0.23.2 → tokenizers-0.23.3}/py_src/tokenizers/tokenizers.pyi +0 -0
  88. {tokenizers-0.23.2 → tokenizers-0.23.3}/py_src/tokenizers/tools/__init__.py +0 -0
  89. {tokenizers-0.23.2 → tokenizers-0.23.3}/py_src/tokenizers/tools/visualizer-styles.css +0 -0
  90. {tokenizers-0.23.2 → tokenizers-0.23.3}/py_src/tokenizers/tools/visualizer.py +0 -0
  91. {tokenizers-0.23.2 → tokenizers-0.23.3}/py_src/tokenizers/trainers/__init__.py +0 -0
  92. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/CHANGELOG.md +0 -0
  93. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/Cargo.toml +0 -0
  94. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/LICENSE +0 -0
  95. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/Makefile +0 -0
  96. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/README.md +0 -0
  97. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/README.tpl +0 -0
  98. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/benches/added_vocab_deserialize.rs +0 -0
  99. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/benches/bert_benchmark.rs +0 -0
  100. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/benches/bpe_benchmark.rs +0 -0
  101. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/benches/ci_benchmark.rs +0 -0
  102. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/benches/common/mod.rs +0 -0
  103. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/benches/decode_benchmark.rs +0 -0
  104. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/benches/layout_benchmark.rs +0 -0
  105. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/benches/llama3_benchmark.rs +0 -0
  106. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/benches/truncation_benchmark.rs +0 -0
  107. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/benches/unigram_benchmark.rs +0 -0
  108. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/decoders/bpe.rs +0 -0
  109. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/decoders/byte_fallback.rs +0 -0
  110. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/decoders/ctc.rs +0 -0
  111. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/decoders/fuse.rs +0 -0
  112. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/decoders/mod.rs +0 -0
  113. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/decoders/sequence.rs +0 -0
  114. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/decoders/strip.rs +0 -0
  115. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/decoders/wordpiece.rs +0 -0
  116. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/lib.rs +0 -0
  117. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/models/bpe/mod.rs +0 -0
  118. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/models/bpe/model.rs +0 -0
  119. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/models/bpe/parity_trainer.rs +0 -0
  120. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/models/bpe/serialization.rs +0 -0
  121. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/models/bpe/trainer.rs +0 -0
  122. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/models/bpe/word.rs +0 -0
  123. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/models/mod.rs +0 -0
  124. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/models/unigram/lattice.rs +0 -0
  125. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/models/unigram/mod.rs +0 -0
  126. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/models/unigram/model.rs +0 -0
  127. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/models/unigram/serialization.rs +0 -0
  128. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/models/unigram/trainer.rs +0 -0
  129. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/models/unigram/trie.rs +0 -0
  130. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/models/wordlevel/mod.rs +0 -0
  131. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/models/wordlevel/serialization.rs +0 -0
  132. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/models/wordlevel/trainer.rs +0 -0
  133. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/models/wordpiece/mod.rs +0 -0
  134. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/models/wordpiece/serialization.rs +0 -0
  135. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/models/wordpiece/trainer.rs +0 -0
  136. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/normalizers/bert.rs +0 -0
  137. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/normalizers/byte_level.rs +0 -0
  138. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/normalizers/mod.rs +0 -0
  139. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/normalizers/precompiled.rs +0 -0
  140. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/normalizers/prepend.rs +0 -0
  141. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/normalizers/replace.rs +0 -0
  142. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/normalizers/strip.rs +0 -0
  143. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/normalizers/unicode.rs +0 -0
  144. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/normalizers/utils.rs +0 -0
  145. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/pre_tokenizers/bert.rs +0 -0
  146. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/pre_tokenizers/byte_level.rs +0 -0
  147. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/pre_tokenizers/delimiter.rs +0 -0
  148. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/pre_tokenizers/digits.rs +0 -0
  149. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/pre_tokenizers/fixed_length.rs +0 -0
  150. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/pre_tokenizers/metaspace.rs +0 -0
  151. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/pre_tokenizers/mod.rs +0 -0
  152. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/pre_tokenizers/punctuation.rs +0 -0
  153. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/pre_tokenizers/sequence.rs +0 -0
  154. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/pre_tokenizers/split.rs +0 -0
  155. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/pre_tokenizers/unicode_scripts/mod.rs +0 -0
  156. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/pre_tokenizers/unicode_scripts/pre_tokenizer.rs +0 -0
  157. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/pre_tokenizers/unicode_scripts/scripts.rs +0 -0
  158. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/pre_tokenizers/whitespace.rs +0 -0
  159. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/processors/bert.rs +0 -0
  160. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/processors/mod.rs +0 -0
  161. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/processors/roberta.rs +0 -0
  162. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/processors/sequence.rs +0 -0
  163. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/processors/template.rs +0 -0
  164. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/tokenizer/added_vocabulary.rs +0 -0
  165. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/tokenizer/encoding.rs +0 -0
  166. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/tokenizer/normalizer.rs +0 -0
  167. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/tokenizer/pattern.rs +0 -0
  168. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/tokenizer/serialization.rs +0 -0
  169. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/utils/cache.rs +0 -0
  170. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/utils/fancy.rs +0 -0
  171. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/utils/from_pretrained.rs +0 -0
  172. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/utils/iter.rs +0 -0
  173. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/utils/mod.rs +0 -0
  174. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/utils/onig.rs +0 -0
  175. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/utils/padding.rs +0 -0
  176. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/utils/parallelism.rs +0 -0
  177. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/utils/progress.rs +0 -0
  178. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/utils/truncation.rs +0 -0
  179. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/tests/added_tokens.rs +0 -0
  180. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/tests/common/mod.rs +0 -0
  181. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/tests/documentation.rs +0 -0
  182. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/tests/from_pretrained.rs +0 -0
  183. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/tests/offsets.rs +0 -0
  184. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/tests/serialization.rs +0 -0
  185. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/tests/stream.rs +0 -0
  186. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/tests/training.rs +0 -0
  187. {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/tests/unigram.rs +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: tokenizers
3
- Version: 0.23.2
3
+ Version: 0.23.3
4
4
  Classifier: Development Status :: 5 - Production/Stable
5
5
  Classifier: Intended Audience :: Developers
6
6
  Classifier: Intended Audience :: Education
@@ -15,7 +15,7 @@ Classifier: Programming Language :: Python :: 3.13
15
15
  Classifier: Programming Language :: Python :: 3.14
16
16
  Classifier: Programming Language :: Python :: 3 :: Only
17
17
  Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
18
- Requires-Dist: huggingface-hub>=0.16.4,<2.0
18
+ Requires-Dist: huggingface-hub>=0.16.4,<3.0
19
19
  Requires-Dist: tokenizers[testing] ; extra == 'dev'
20
20
  Requires-Dist: sphinx ; extra == 'docs'
21
21
  Requires-Dist: sphinx-rtd-theme ; extra == 'docs'
@@ -1204,7 +1204,7 @@ dependencies = [
1204
1204
 
1205
1205
  [[package]]
1206
1206
  name = "tokenizers-python"
1207
- version = "0.23.2"
1207
+ version = "0.23.3"
1208
1208
  dependencies = [
1209
1209
  "ahash",
1210
1210
  "env_logger",
@@ -1,6 +1,6 @@
1
1
  [package]
2
2
  name = "tokenizers-python"
3
- version = "0.23.2"
3
+ version = "0.23.3"
4
4
  authors = ["Anthony MOI <m.anthony.moi@gmail.com>"]
5
5
  edition = "2021"
6
6
  readme = "README.md"
@@ -34,13 +34,15 @@ check-style:
34
34
  $(CARGO_ENV) cargo run --manifest-path ./tools/stub-gen/Cargo.toml
35
35
  ruff check $(check_stubs) --select I --fix
36
36
  ruff check $(check_dirs)
37
- ruff format py_src/tokenizers/*.pyi
37
+ ruff format $(check_stubs)
38
38
  ruff format --check $(check_dirs)
39
39
  ty check py_src --exclude py_src/tokenizers/implementations --exclude py_src/tokenizers/tools/visualizer.py
40
40
 
41
41
 
42
42
  TESTS_RESOURCES = $(DATA_DIR)/small.txt $(DATA_DIR)/roberta.json
43
43
 
44
+ test-data: $(TESTS_RESOURCES)
45
+
44
46
  # Launch the Python test suite
45
47
  test-py: $(TESTS_RESOURCES)
46
48
  $(PIP) install pytest pytest-asyncio huggingface_hub setuptools_rust numpy pyarrow datasets
@@ -53,20 +53,9 @@ impl Model for PyModel {
53
53
  }
54
54
 
55
55
  /// See [`Model::tokenize_in_pretokenized`] for the lock-once rationale.
56
- fn tokenize_in_pretokenized(
57
- &self,
58
- pretokenized: &mut PreTokenizedString,
59
- truncation: Option<(usize, tk::TruncationDirection)>,
60
- ) -> tk::Result<()> {
56
+ fn tokenize_in_pretokenized(&self, pretokenized: &mut PreTokenizedString) -> tk::Result<()> {
61
57
  let guard = self.model.read().unwrap();
62
- match truncation {
63
- Some((max_tokens, direction)) => pretokenized.tokenize_with_limit(
64
- |normalized| guard.tokenize(normalized.get()),
65
- max_tokens,
66
- direction,
67
- ),
68
- None => pretokenized.tokenize(|normalized| guard.tokenize(normalized.get())),
69
- }
58
+ pretokenized.tokenize(|normalized| guard.tokenize(normalized.get()))
70
59
  }
71
60
 
72
61
  fn token_to_id(&self, token: &str) -> Option<u32> {
@@ -361,7 +361,7 @@ impl PyBpeTrainer {
361
361
  #[new]
362
362
  #[pyo3(
363
363
  signature = (**kwargs),
364
- text_signature = "(self, vocab_size=30000, min_frequency=0, show_progress=True, progress_format=\"indicatif\", special_tokens=[], limit_alphabet=None, initial_alphabet=[], continuing_subword_prefix=None, end_of_word_suffix=None, max_token_length=None, words={})"
364
+ text_signature = "(self, vocab_size=30000, min_frequency=0, show_progress=True, progress_format=\"indicatif\", special_tokens=[], limit_alphabet=None, initial_alphabet=[], continuing_subword_prefix=None, end_of_word_suffix=None, max_token_length=None)"
365
365
  )]
366
366
  pub fn new(kwargs: Option<&Bound<'_, PyDict>>) -> PyResult<PyClassInitializer<Self>> {
367
367
  let mut builder = tk::models::bpe::BpeTrainer::builder();
@@ -51,14 +51,14 @@ class AddedToken:
51
51
  special (:obj:`bool`, defaults to :obj:`False` with :meth:`~tokenizers.Tokenizer.add_tokens` and :obj:`False` with :meth:`~tokenizers.Tokenizer.add_special_tokens`):
52
52
  Defines whether this token should be skipped when decoding.
53
53
  """
54
- def __eq__(self, /, other: object) -> bool: ...
55
- def __ge__(self, /, other: object) -> bool: ...
54
+ def __eq__(self, other: object, /) -> bool: ...
55
+ def __ge__(self, other: object, /) -> bool: ...
56
56
  def __getstate__(self, /) -> dict: ...
57
- def __gt__(self, /, other: object) -> bool: ...
57
+ def __gt__(self, other: object, /) -> bool: ...
58
58
  def __hash__(self, /) -> int: ...
59
- def __le__(self, /, other: object) -> bool: ...
60
- def __lt__(self, /, other: object) -> bool: ...
61
- def __ne__(self, /, other: object) -> bool: ...
59
+ def __le__(self, other: object, /) -> bool: ...
60
+ def __lt__(self, other: object, /) -> bool: ...
61
+ def __ne__(self, other: object, /) -> bool: ...
62
62
  def __new__(cls, /, content: str | None = None, **kwargs) -> AddedToken: ...
63
63
  def __repr__(self, /) -> str: ...
64
64
  def __setstate__(self, /, state: Any) -> None: ...
@@ -435,7 +435,7 @@ class NormalizedString:
435
435
  sequence: str:
436
436
  The string sequence used to initialize this NormalizedString
437
437
  """
438
- def __getitem__(self, /, range: int | tuple[int, int] | slice) -> NormalizedString | None: ...
438
+ def __getitem__(self, range: int | tuple[int, int] | slice, /) -> NormalizedString | None: ...
439
439
  def __new__(cls, /, sequence: str) -> NormalizedString: ...
440
440
  def __repr__(self, /) -> str: ...
441
441
  def __str__(self, /) -> str: ...
@@ -70,7 +70,7 @@ class ByteLevel(Decoder):
70
70
  >>> decoder.decode(["ĠHello", "Ġworld"])
71
71
  ' Hello world'
72
72
  """
73
- def __new__(cls, /, **_kwargs) -> ByteLevel: ...
73
+ def __new__(cls, /) -> ByteLevel: ...
74
74
 
75
75
  @final
76
76
  class CTC(Decoder):
@@ -80,7 +80,7 @@ class ByteLevel(Normalizer):
80
80
 
81
81
  >>> from tokenizers.normalizers import ByteLevel
82
82
  >>> normalizer = ByteLevel()
83
- >>> normalizer.normalize_str("hello\nworld")
83
+ >>> normalizer.normalize_str("hello\\nworld")
84
84
  'helloĊworld'
85
85
  """
86
86
  def __new__(cls, /) -> ByteLevel: ...
@@ -115,7 +115,7 @@ class NFC(Normalizer):
115
115
 
116
116
  >>> from tokenizers.normalizers import NFC
117
117
  >>> normalizer = NFC()
118
- >>> normalizer.normalize_str("e\u0301") # 'e' + combining accent
118
+ >>> normalizer.normalize_str("e\\u0301") # 'e' + combining accent
119
119
  'é'
120
120
  """
121
121
  def __new__(cls, /) -> NFC: ...
@@ -137,7 +137,7 @@ class NFD(Normalizer):
137
137
  >>> from tokenizers.normalizers import NFD
138
138
  >>> normalizer = NFD()
139
139
  >>> normalizer.normalize_str("Héllo")
140
- 'He\u0301llo'
140
+ 'He\\u0301llo'
141
141
  """
142
142
  def __new__(cls, /) -> NFD: ...
143
143
 
@@ -155,7 +155,7 @@ class NFKC(Normalizer):
155
155
 
156
156
  >>> from tokenizers.normalizers import NFKC
157
157
  >>> normalizer = NFKC()
158
- >>> normalizer.normalize_str("fine caf\u00e9")
158
+ >>> normalizer.normalize_str("fine caf\\u00e9")
159
159
  'fine café'
160
160
  """
161
161
  def __new__(cls, /) -> NFKC: ...
@@ -192,7 +192,7 @@ class Nmt(Normalizer):
192
192
 
193
193
  >>> from tokenizers.normalizers import Nmt
194
194
  >>> normalizer = Nmt()
195
- >>> normalizer.normalize_str("Hello\x00World")
195
+ >>> normalizer.normalize_str("Hello\\x00World")
196
196
  'Hello World'
197
197
  """
198
198
  def __new__(cls, /) -> Nmt: ...
@@ -310,7 +310,7 @@ class Replace(Normalizer):
310
310
  >>> Replace(".", " ").normalize_str("hello.world")
311
311
  'hello world'
312
312
  >>> # Replace using a regex
313
- >>> Replace(Regex(r"\s+"), " ").normalize_str("hello world")
313
+ >>> Replace(Regex(r"\\s+"), " ").normalize_str("hello world")
314
314
  'hello world'
315
315
  """
316
316
  def __new__(cls, /, pattern: str | Regex, content: str) -> Replace: ...
@@ -340,11 +340,11 @@ class Sequence(Normalizer):
340
340
  >>> normalizer.normalize_str("Héllo Wörld")
341
341
  'hello world'
342
342
  """
343
- def __getitem__(self, /, index: int) -> Any: ...
343
+ def __getitem__(self, index: int, /) -> Any: ...
344
344
  def __getnewargs__(self, /) -> tuple: ...
345
345
  def __len__(self, /) -> int: ...
346
346
  def __new__(cls, /, normalizers: list) -> Sequence: ...
347
- def __setitem__(self, /, index: int, value: Any) -> None: ...
347
+ def __setitem__(self, index: int, value: Any, /) -> None: ...
348
348
 
349
349
  @final
350
350
  class Strip(Normalizer):
@@ -279,10 +279,10 @@ class Sequence(PreTokenizer):
279
279
  >>> pre_tokenizer.pre_tokenize_str("Hello, world!")
280
280
  [('Hello', (0, 5)), (',', (5, 6)), ('world', (7, 12)), ('!', (12, 13))]
281
281
  """
282
- def __getitem__(self, /, index: int) -> Any: ...
282
+ def __getitem__(self, index: int, /) -> Any: ...
283
283
  def __getnewargs__(self, /) -> tuple: ...
284
284
  def __new__(cls, /, pre_tokenizers: list) -> Sequence: ...
285
- def __setitem__(self, /, index: int, value: Any) -> None: ...
285
+ def __setitem__(self, index: int, value: Any, /) -> None: ...
286
286
 
287
287
  @final
288
288
  class Split(PreTokenizer):
@@ -317,7 +317,7 @@ class Split(PreTokenizer):
317
317
  >>> pre_tokenizer.pre_tokenize_str("one,two,three")
318
318
  [('one', (0, 3)), ('two', (4, 7)), ('three', (8, 13))]
319
319
  >>> # Split using a regex, keeping the delimiter isolated
320
- >>> Split(Regex(r"\s+"), behavior="isolated").pre_tokenize_str("hello world")
320
+ >>> Split(Regex(r"\\s+"), behavior="isolated").pre_tokenize_str("hello world")
321
321
  [('hello', (0, 5)), (' ', (5, 8)), ('world', (8, 13))]
322
322
  """
323
323
  def __getnewargs__(self, /) -> tuple: ...
@@ -355,7 +355,7 @@ class UnicodeScripts(PreTokenizer):
355
355
  @final
356
356
  class Whitespace(PreTokenizer):
357
357
  """
358
- This pre-tokenizer splits on word boundaries according to the ``\w+|[^\w\s]+``
358
+ This pre-tokenizer splits on word boundaries according to the ``\\w+|[^\\w\\s]+``
359
359
  regex pattern. It splits on word characters or characters that aren't words or
360
360
  whitespaces (punctuation such as hyphens, apostrophes, commas, etc.).
361
361
 
@@ -207,10 +207,10 @@ class Sequence(PostProcessor):
207
207
  >>> from tokenizers.processors import BertProcessing, ByteLevel, Sequence
208
208
  >>> processor = Sequence([ByteLevel(trim_offsets=True), BertProcessing(("[SEP]", 102), ("[CLS]", 101))])
209
209
  """
210
- def __getitem__(self, /, index: int) -> Any: ...
210
+ def __getitem__(self, index: int, /) -> Any: ...
211
211
  def __getnewargs__(self, /) -> tuple: ...
212
212
  def __new__(cls, /, processors_py: list) -> Sequence: ...
213
- def __setitem__(self, /, index: int, value: Any) -> None: ...
213
+ def __setitem__(self, index: int, value: Any, /) -> None: ...
214
214
 
215
215
  @final
216
216
  class TemplateProcessing(PostProcessor):
@@ -5,7 +5,7 @@ Trainers Module
5
5
  from collections.abc import Sequence
6
6
  from typing import Any, final
7
7
 
8
- from tokenizers import AddedToken
8
+ from tokenizers import AddedToken, Tokenizer
9
9
 
10
10
  @final
11
11
  class BpeTrainer(Trainer):
@@ -57,7 +57,21 @@ class BpeTrainer(Trainer):
57
57
  >>> tokenizer = Tokenizer(BPE())
58
58
  >>> tokenizer.train(["path/to/corpus.txt"], trainer)
59
59
  """
60
- def __new__(cls, /, **kwargs) -> BpeTrainer: ...
60
+ def __new__(
61
+ cls,
62
+ /,
63
+ *,
64
+ vocab_size: int = 30000,
65
+ min_frequency: int = 0,
66
+ show_progress: bool = True,
67
+ progress_format: str = "indicatif",
68
+ special_tokens: list = [],
69
+ limit_alphabet: int | None = None,
70
+ initial_alphabet: Sequence[str] = [],
71
+ continuing_subword_prefix: str | None = None,
72
+ end_of_word_suffix: str | None = None,
73
+ max_token_length: int | None = None,
74
+ ) -> BpeTrainer: ...
61
75
  @property
62
76
  def continuing_subword_prefix(self, /) -> str | None: ...
63
77
  @continuing_subword_prefix.setter
@@ -111,6 +125,75 @@ class BpeTrainer(Trainer):
111
125
 
112
126
  @final
113
127
  class ParityBpeTrainer:
128
+ """
129
+ Trainer for parity-aware BPE that ensures cross-lingual fairness in tokenization.
130
+
131
+ Unlike standard BPE, this trainer takes one Python iterator per language and
132
+ balances merge operations across languages using a development set or target
133
+ compression ratios. The single training entry point is
134
+ :meth:`train_from_iterator`, the multi-corpus analogue of
135
+ :meth:`tokenizers.Tokenizer.train_from_iterator`. Balancing requires either
136
+ ``dev_iterators`` (ideally parallel across languages) or ``ratio``; given
137
+ neither, selection follows the per-language training totals, which the
138
+ highest-data language dominates, so the result stays close to plain BPE. When
139
+ both are given the dev set takes precedence. A runnable version that obtains
140
+ per-language corpora and a parallel dev set is in
141
+ ``examples/train_parity_bpe.py``.
142
+
143
+ Args:
144
+ num_merges (:obj:`int`, `optional`):
145
+ Number of BPE merge operations to perform. Defaults to ``32000``.
146
+
147
+ variant (:obj:`str`, `optional`):
148
+ Algorithm variant: ``"base"`` (default) or ``"window"`` (moving-window balancing).
149
+
150
+ min_frequency (:obj:`int`, `optional`):
151
+ Minimum pair frequency to merge. Defaults to ``0``.
152
+
153
+ ratio (:obj:`List[float]`, `optional`):
154
+ Target compression rate per language, one entry per training iterator. The
155
+ trainer selects the language with the lowest ``compression_rate / ratio``,
156
+ so raising one language's ratio gives it more merges; only the values
157
+ relative to each other matter. Rates are counted in the units the
158
+ pre-tokenizer emits, bytes under
159
+ :class:`~tokenizers.pre_tokenizers.ByteLevel`, so equal ratios do not give
160
+ equal tokenization across scripts: Devanagari takes about 2.5x the bytes of
161
+ Latin script for the same content. Set each ratio proportional to the
162
+ language's average length on parallel text instead. Defaults to ``None``.
163
+
164
+ global_merges (:obj:`int`, `optional`):
165
+ Number of initial standard BPE merges before switching to parity mode. Defaults to ``0``.
166
+
167
+ window_size (:obj:`int`, `optional`):
168
+ Window size for the ``"window"`` variant. Defaults to ``100``.
169
+
170
+ alpha (:obj:`float`, `optional`):
171
+ Alpha parameter for the ``"window"`` variant. Defaults to ``2.0``.
172
+
173
+ total_symbols (:obj:`bool`, `optional`):
174
+ If True, subtract unique character count from ``num_merges``. Defaults to ``False``.
175
+
176
+ Example::
177
+
178
+ >>> from tokenizers import Tokenizer, pre_tokenizers
179
+ >>> from tokenizers.models import BPE
180
+ >>> from tokenizers.trainers import ParityBpeTrainer
181
+ >>> tokenizer = Tokenizer(BPE())
182
+ >>> tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel()
183
+ >>> trainer = ParityBpeTrainer(num_merges=32000, variant="base")
184
+ >>> # balance against a parallel dev set, the same sentences in every language
185
+ >>> trainer.train_from_iterator(
186
+ ... tokenizer,
187
+ ... train_iterators=[english_lines, hindi_lines],
188
+ ... dev_iterators=[english_dev, hindi_dev],
189
+ ... )
190
+ >>> # or against target rates, which need no dev data
191
+ >>> trainer.train_from_iterator(
192
+ ... tokenizer,
193
+ ... train_iterators=[english_lines, hindi_lines],
194
+ ... ratio=[1.0, 2.57],
195
+ ... )
196
+ """
114
197
  def __getstate__(self, /) -> Any: ...
115
198
  def __new__(
116
199
  cls,
@@ -130,16 +213,10 @@ class ParityBpeTrainer:
130
213
  continuing_subword_prefix: str | None = None,
131
214
  end_of_word_suffix: str | None = None,
132
215
  max_token_length: int | None = None,
133
- ) -> ParityBpeTrainer:
134
- """Create and return a new object. See help(type) for accurate signature."""
135
- ...
136
- def __repr__(self, /) -> str:
137
- """Return repr(self)."""
138
- ...
216
+ ) -> ParityBpeTrainer: ...
217
+ def __repr__(self, /) -> str: ...
139
218
  def __setstate__(self, /, state: Any) -> None: ...
140
- def __str__(self, /) -> str:
141
- """Return str(self)."""
142
- ...
219
+ def __str__(self, /) -> str: ...
143
220
  @property
144
221
  def alpha(self, /) -> float: ...
145
222
  @alpha.setter
@@ -188,6 +265,44 @@ class ParityBpeTrainer:
188
265
  def total_symbols(self, /) -> bool: ...
189
266
  @total_symbols.setter
190
267
  def total_symbols(self, /, v: bool) -> None: ...
268
+ def train_from_iterator(
269
+ self,
270
+ /,
271
+ tokenizer: Tokenizer,
272
+ train_iterators: Sequence[Any],
273
+ dev_iterators: Sequence[Any] | None = None,
274
+ ratio: Sequence[float] | None = None,
275
+ ) -> None:
276
+ """
277
+ Train a user-configured tokenizer with parity-aware BPE from per-language
278
+ Python iterators.
279
+
280
+ Each entry of ``train_iterators`` (and optionally ``dev_iterators``) is a
281
+ Python iterator yielding strings (or batches / lists of strings) for one
282
+ language. This is the multi-corpus analogue of
283
+ :meth:`~tokenizers.Tokenizer.train_from_iterator`: file I/O happens in
284
+ Python, so users can pull data from plain text, parquet (via ``pyarrow``),
285
+ ``datasets``, etc.
286
+
287
+ Args:
288
+ tokenizer (:class:`~tokenizers.Tokenizer`):
289
+ A tokenizer instance to train. Its pre-tokenizer (and optionally
290
+ normalizer) should already be configured.
291
+
292
+ train_iterators (:obj:`List[Iterator]`):
293
+ One Python iterator per language, each yielding ``str`` or
294
+ ``List[str]``.
295
+
296
+ dev_iterators (:obj:`List[Iterator]`, `optional`):
297
+ One Python iterator per language, used to drive parity-aware
298
+ language selection. Must have the same length as
299
+ ``train_iterators``.
300
+
301
+ ratio (:obj:`List[float]`, `optional`):
302
+ Target compression rates per language, an alternative to
303
+ ``dev_iterators`` and ignored when one is supplied. See the class
304
+ docstring for how to choose the values.
305
+ """
191
306
  @property
192
307
  def variant(self, /) -> str: ...
193
308
  @property
@@ -254,7 +369,19 @@ class UnigramTrainer(Trainer):
254
369
  >>> tokenizer = Tokenizer(Unigram())
255
370
  >>> tokenizer.train(["path/to/corpus.txt"], trainer)
256
371
  """
257
- def __new__(cls, /, **kwargs) -> UnigramTrainer: ...
372
+ def __new__(
373
+ cls,
374
+ /,
375
+ *,
376
+ vocab_size: int = 8000,
377
+ show_progress: bool = True,
378
+ special_tokens: list = [],
379
+ initial_alphabet: Sequence[str] = [],
380
+ shrinking_factor=0.75,
381
+ unk_token=None,
382
+ max_piece_length=16,
383
+ n_sub_iterations=2,
384
+ ) -> UnigramTrainer: ...
258
385
  @property
259
386
  def initial_alphabet(self, /) -> list[str]: ...
260
387
  @initial_alphabet.setter
@@ -302,7 +429,15 @@ class WordLevelTrainer(Trainer):
302
429
  >>> tokenizer = Tokenizer(WordLevel(unk_token="<unk>"))
303
430
  >>> tokenizer.train(["path/to/corpus.txt"], trainer)
304
431
  """
305
- def __new__(cls, /, **kwargs) -> WordLevelTrainer: ...
432
+ def __new__(
433
+ cls,
434
+ /,
435
+ *,
436
+ vocab_size: int = 30000,
437
+ min_frequency: int = 0,
438
+ show_progress: bool = True,
439
+ special_tokens: list = [],
440
+ ) -> WordLevelTrainer: ...
306
441
  @property
307
442
  def min_frequency(self, /) -> int: ...
308
443
  @min_frequency.setter
@@ -364,7 +499,19 @@ class WordPieceTrainer(Trainer):
364
499
  >>> tokenizer = Tokenizer(WordPiece(unk_token="[UNK]"))
365
500
  >>> tokenizer.train(["path/to/corpus.txt"], trainer)
366
501
  """
367
- def __new__(cls, /, **kwargs) -> WordPieceTrainer: ...
502
+ def __new__(
503
+ cls,
504
+ /,
505
+ *,
506
+ vocab_size: int = 30000,
507
+ min_frequency: int = 0,
508
+ show_progress: bool = True,
509
+ special_tokens: list = [],
510
+ limit_alphabet: int | None = None,
511
+ initial_alphabet: Sequence[str] = [],
512
+ continuing_subword_prefix: str | None = "##",
513
+ end_of_word_suffix: str | None = None,
514
+ ) -> WordPieceTrainer: ...
368
515
  @property
369
516
  def continuing_subword_prefix(self, /) -> str | None: ...
370
517
  @continuing_subword_prefix.setter
@@ -23,7 +23,7 @@ classifiers = [
23
23
  ]
24
24
  keywords = ["NLP", "tokenizer", "BPE", "transformer", "deep learning"]
25
25
  dynamic = ["description", "license", "readme", "version"]
26
- dependencies = ["huggingface_hub>=0.16.4,<2.0"]
26
+ dependencies = ["huggingface_hub>=0.16.4,<3.0"]
27
27
 
28
28
  [project.urls]
29
29
  Homepage = "https://github.com/huggingface/tokenizers"
@@ -88,32 +88,15 @@ pub trait Model {
88
88
 
89
89
  /// Tokenize every pre-token within a `PreTokenizedString` in one call.
90
90
  ///
91
- /// When `truncation` is `Some((max_tokens, direction))`, the
92
- /// `tokenize_with_limit` early-exit path is taken; otherwise every
93
- /// pre-token is tokenized.
94
- ///
95
91
  /// The default calls `self.tokenize()` per pre-token, which is correct
96
92
  /// for self-contained `Model` implementations. Implementations that
97
93
  /// wrap their inner model behind a lock (e.g. the `PyModel` and Node
98
94
  /// `Model` bindings, both `Arc<RwLock<_>>`) can override this to
99
95
  /// acquire the lock once for the whole sequence of pre-tokens; that
100
96
  /// removes ~one atomic load/store pair per pre-token from the hot path
101
- /// (~1 500 pre-tokens per ~10 KB document). Both the truncated and
102
- /// non-truncated paths benefit from the override because they share
103
- /// this entry point.
104
- fn tokenize_in_pretokenized(
105
- &self,
106
- pretokenized: &mut PreTokenizedString,
107
- truncation: Option<(usize, TruncationDirection)>,
108
- ) -> Result<()> {
109
- match truncation {
110
- Some((max_tokens, direction)) => pretokenized.tokenize_with_limit(
111
- |normalized| self.tokenize(normalized.get()),
112
- max_tokens,
113
- direction,
114
- ),
115
- None => pretokenized.tokenize(|normalized| self.tokenize(normalized.get())),
116
- }
97
+ /// (~1 500 pre-tokens per ~10 KB document).
98
+ fn tokenize_in_pretokenized(&self, pretokenized: &mut PreTokenizedString) -> Result<()> {
99
+ pretokenized.tokenize(|normalized| self.tokenize(normalized.get()))
117
100
  }
118
101
  }
119
102
 
@@ -1183,19 +1166,7 @@ where
1183
1166
  offsets_type: OffsetType,
1184
1167
  ) -> Result<Encoding> {
1185
1168
  let mut pretokenized: PreTokenizedString = pretokenized.into();
1186
- let truncation = match self.truncation.as_ref() {
1187
- Some(TruncationParams {
1188
- direction,
1189
- max_length,
1190
- strategy,
1191
- ..
1192
- }) if *strategy != TruncationStrategy::OnlySecond || type_id != 0 => {
1193
- Some((*max_length, *direction))
1194
- }
1195
- _ => None,
1196
- };
1197
- self.model
1198
- .tokenize_in_pretokenized(&mut pretokenized, truncation)?;
1169
+ self.model.tokenize_in_pretokenized(&mut pretokenized)?;
1199
1170
  pretokenized.into_encoding(word_idx, type_id, offsets_type)
1200
1171
  }
1201
1172
  }
@@ -1666,7 +1637,7 @@ mod tests {
1666
1637
  }
1667
1638
 
1668
1639
  #[test]
1669
- fn right_truncation_early_exit_matches_full_encode() {
1640
+ fn right_truncation_matches_full_encode() {
1670
1641
  // "a b c d e f g h i j" → 10 tokens [0,1,2,3,4,5,6,7,8,9]
1671
1642
  // Right truncation to 3 → [0,1,2]
1672
1643
  let input = "a b c d e f g h i j";
@@ -1692,6 +1663,61 @@ mod tests {
1692
1663
  );
1693
1664
  }
1694
1665
 
1666
+ fn overflowing_ids(encoding: &Encoding) -> Vec<Vec<u32>> {
1667
+ encoding
1668
+ .get_overflowing()
1669
+ .iter()
1670
+ .map(|o| o.get_ids().to_vec())
1671
+ .collect()
1672
+ }
1673
+
1674
+ #[test]
1675
+ fn truncation_keeps_every_overflowing_chunk() {
1676
+ let input = "a b c d e f g h i j";
1677
+ for direction in [TruncationDirection::Right, TruncationDirection::Left] {
1678
+ for stride in [0, 1] {
1679
+ let mut tok = test_tokenizer();
1680
+ tok.with_truncation(Some(TruncationParams {
1681
+ max_length: 3,
1682
+ strategy: TruncationStrategy::LongestFirst,
1683
+ stride,
1684
+ direction,
1685
+ }))
1686
+ .unwrap();
1687
+ let truncated = tok.encode(input, false).unwrap();
1688
+
1689
+ let mut expected = test_tokenizer().encode(input, false).unwrap();
1690
+ expected.truncate(3, stride, direction);
1691
+
1692
+ assert_eq!(truncated.get_ids(), expected.get_ids());
1693
+ assert_eq!(
1694
+ overflowing_ids(&truncated),
1695
+ overflowing_ids(&expected),
1696
+ "{direction:?}, stride {stride}"
1697
+ );
1698
+ }
1699
+ }
1700
+ }
1701
+
1702
+ #[test]
1703
+ fn right_truncation_overflow_covers_the_whole_input() {
1704
+ let mut tok = test_tokenizer();
1705
+ tok.with_truncation(Some(TruncationParams {
1706
+ max_length: 3,
1707
+ strategy: TruncationStrategy::LongestFirst,
1708
+ stride: 0,
1709
+ direction: TruncationDirection::Right,
1710
+ }))
1711
+ .unwrap();
1712
+ let encoding = tok.encode("a b c d e f g h i j", false).unwrap();
1713
+
1714
+ assert_eq!(encoding.get_ids(), &[0, 1, 2]);
1715
+ assert_eq!(
1716
+ overflowing_ids(&encoding),
1717
+ vec![vec![3, 4, 5], vec![6, 7, 8], vec![9]]
1718
+ );
1719
+ }
1720
+
1695
1721
  #[test]
1696
1722
  fn left_truncation_keeps_tail_tokens() {
1697
1723
  // "a b c d e f g h i j" → 10 tokens [0,1,2,3,4,5,6,7,8,9]