tokenizers 0.21.2__tar.gz → 0.21.4.dev0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of tokenizers might be problematic. Click here for more details.

Files changed (207) hide show
  1. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/PKG-INFO +1 -1
  2. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/bindings/python/Cargo.lock +138 -65
  3. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/bindings/python/Cargo.toml +1 -1
  4. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/bindings/python/src/decoders.rs +2 -4
  5. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/bindings/python/src/encoding.rs +9 -13
  6. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/bindings/python/src/error.rs +3 -3
  7. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/bindings/python/src/lib.rs +1 -2
  8. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/bindings/python/src/models.rs +17 -25
  9. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/bindings/python/src/normalizers.rs +5 -8
  10. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/bindings/python/src/pre_tokenizers.rs +3 -6
  11. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/bindings/python/src/processors.rs +2 -4
  12. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/bindings/python/src/tokenizer.rs +12 -18
  13. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/bindings/python/src/trainers.rs +7 -9
  14. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/bindings/python/src/utils/normalization.rs +1 -2
  15. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/Cargo.toml +1 -1
  16. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/src/models/bpe/trainer.rs +5 -2
  17. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/src/models/wordpiece/trainer.rs +7 -3
  18. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/bindings/python/.cargo/config.toml +0 -0
  19. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/bindings/python/.gitignore +0 -0
  20. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/bindings/python/CHANGELOG.md +0 -0
  21. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/bindings/python/MANIFEST.in +0 -0
  22. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/bindings/python/Makefile +0 -0
  23. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/bindings/python/README.md +0 -0
  24. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/bindings/python/benches/test_tiktoken.py +0 -0
  25. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/bindings/python/conftest.py +0 -0
  26. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/bindings/python/examples/custom_components.py +0 -0
  27. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/bindings/python/examples/example.py +0 -0
  28. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/bindings/python/examples/train_bert_wordpiece.py +0 -0
  29. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/bindings/python/examples/train_bytelevel_bpe.py +0 -0
  30. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/bindings/python/examples/train_with_datasets.py +0 -0
  31. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/bindings/python/examples/using_the_visualizer.ipynb +0 -0
  32. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/bindings/python/py_src/tokenizers/__init__.py +0 -0
  33. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/bindings/python/py_src/tokenizers/__init__.pyi +0 -0
  34. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/bindings/python/py_src/tokenizers/decoders/__init__.py +0 -0
  35. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/bindings/python/py_src/tokenizers/decoders/__init__.pyi +0 -0
  36. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/bindings/python/py_src/tokenizers/implementations/__init__.py +0 -0
  37. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/bindings/python/py_src/tokenizers/implementations/base_tokenizer.py +0 -0
  38. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/bindings/python/py_src/tokenizers/implementations/bert_wordpiece.py +0 -0
  39. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/bindings/python/py_src/tokenizers/implementations/byte_level_bpe.py +0 -0
  40. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/bindings/python/py_src/tokenizers/implementations/char_level_bpe.py +0 -0
  41. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/bindings/python/py_src/tokenizers/implementations/sentencepiece_bpe.py +0 -0
  42. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/bindings/python/py_src/tokenizers/implementations/sentencepiece_unigram.py +0 -0
  43. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/bindings/python/py_src/tokenizers/models/__init__.py +0 -0
  44. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/bindings/python/py_src/tokenizers/models/__init__.pyi +0 -0
  45. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/bindings/python/py_src/tokenizers/normalizers/__init__.py +0 -0
  46. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/bindings/python/py_src/tokenizers/normalizers/__init__.pyi +0 -0
  47. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/bindings/python/py_src/tokenizers/pre_tokenizers/__init__.py +0 -0
  48. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/bindings/python/py_src/tokenizers/pre_tokenizers/__init__.pyi +0 -0
  49. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/bindings/python/py_src/tokenizers/processors/__init__.py +0 -0
  50. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/bindings/python/py_src/tokenizers/processors/__init__.pyi +0 -0
  51. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/bindings/python/py_src/tokenizers/tools/__init__.py +0 -0
  52. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/bindings/python/py_src/tokenizers/tools/visualizer-styles.css +0 -0
  53. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/bindings/python/py_src/tokenizers/tools/visualizer.py +0 -0
  54. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/bindings/python/py_src/tokenizers/trainers/__init__.py +0 -0
  55. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/bindings/python/py_src/tokenizers/trainers/__init__.pyi +0 -0
  56. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/bindings/python/rust-toolchain +0 -0
  57. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/bindings/python/scripts/convert.py +0 -0
  58. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/bindings/python/scripts/sentencepiece_extractor.py +0 -0
  59. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/bindings/python/scripts/spm_parity_check.py +0 -0
  60. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/bindings/python/setup.cfg +0 -0
  61. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/bindings/python/src/token.rs +0 -0
  62. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/bindings/python/src/utils/iterators.rs +0 -0
  63. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/bindings/python/src/utils/mod.rs +0 -0
  64. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/bindings/python/src/utils/pretokenization.rs +0 -0
  65. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/bindings/python/src/utils/regex.rs +0 -0
  66. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/bindings/python/src/utils/serde_pyo3.rs +0 -0
  67. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/bindings/python/stub.py +0 -0
  68. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/bindings/python/test.txt +0 -0
  69. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/bindings/python/tests/__init__.py +0 -0
  70. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/bindings/python/tests/bindings/__init__.py +0 -0
  71. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/bindings/python/tests/bindings/test_decoders.py +0 -0
  72. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/bindings/python/tests/bindings/test_encoding.py +0 -0
  73. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/bindings/python/tests/bindings/test_models.py +0 -0
  74. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/bindings/python/tests/bindings/test_normalizers.py +0 -0
  75. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/bindings/python/tests/bindings/test_pre_tokenizers.py +0 -0
  76. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/bindings/python/tests/bindings/test_processors.py +0 -0
  77. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/bindings/python/tests/bindings/test_tokenizer.py +0 -0
  78. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/bindings/python/tests/bindings/test_trainers.py +0 -0
  79. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/bindings/python/tests/documentation/__init__.py +0 -0
  80. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/bindings/python/tests/documentation/test_pipeline.py +0 -0
  81. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/bindings/python/tests/documentation/test_quicktour.py +0 -0
  82. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/bindings/python/tests/documentation/test_tutorial_train_from_iterators.py +0 -0
  83. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/bindings/python/tests/implementations/__init__.py +0 -0
  84. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/bindings/python/tests/implementations/test_base_tokenizer.py +0 -0
  85. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/bindings/python/tests/implementations/test_bert_wordpiece.py +0 -0
  86. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/bindings/python/tests/implementations/test_byte_level_bpe.py +0 -0
  87. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/bindings/python/tests/implementations/test_char_bpe.py +0 -0
  88. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/bindings/python/tests/implementations/test_sentencepiece.py +0 -0
  89. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/bindings/python/tests/test_serialization.py +0 -0
  90. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/bindings/python/tests/utils.py +0 -0
  91. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/py_src/tokenizers/__init__.py +0 -0
  92. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/py_src/tokenizers/__init__.pyi +0 -0
  93. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/py_src/tokenizers/decoders/__init__.py +0 -0
  94. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/py_src/tokenizers/decoders/__init__.pyi +0 -0
  95. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/py_src/tokenizers/implementations/__init__.py +0 -0
  96. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/py_src/tokenizers/implementations/base_tokenizer.py +0 -0
  97. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/py_src/tokenizers/implementations/bert_wordpiece.py +0 -0
  98. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/py_src/tokenizers/implementations/byte_level_bpe.py +0 -0
  99. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/py_src/tokenizers/implementations/char_level_bpe.py +0 -0
  100. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/py_src/tokenizers/implementations/sentencepiece_bpe.py +0 -0
  101. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/py_src/tokenizers/implementations/sentencepiece_unigram.py +0 -0
  102. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/py_src/tokenizers/models/__init__.py +0 -0
  103. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/py_src/tokenizers/models/__init__.pyi +0 -0
  104. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/py_src/tokenizers/normalizers/__init__.py +0 -0
  105. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/py_src/tokenizers/normalizers/__init__.pyi +0 -0
  106. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/py_src/tokenizers/pre_tokenizers/__init__.py +0 -0
  107. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/py_src/tokenizers/pre_tokenizers/__init__.pyi +0 -0
  108. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/py_src/tokenizers/processors/__init__.py +0 -0
  109. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/py_src/tokenizers/processors/__init__.pyi +0 -0
  110. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/py_src/tokenizers/tools/__init__.py +0 -0
  111. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/py_src/tokenizers/tools/visualizer-styles.css +0 -0
  112. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/py_src/tokenizers/tools/visualizer.py +0 -0
  113. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/py_src/tokenizers/trainers/__init__.py +0 -0
  114. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/py_src/tokenizers/trainers/__init__.pyi +0 -0
  115. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/pyproject.toml +0 -0
  116. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/CHANGELOG.md +0 -0
  117. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/LICENSE +0 -0
  118. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/Makefile +0 -0
  119. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/README.md +0 -0
  120. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/README.tpl +0 -0
  121. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/benches/bert_benchmark.rs +0 -0
  122. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/benches/bpe_benchmark.rs +0 -0
  123. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/benches/common/mod.rs +0 -0
  124. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/benches/layout_benchmark.rs +0 -0
  125. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/benches/llama3_benchmark.rs +0 -0
  126. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/benches/unigram_benchmark.rs +0 -0
  127. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/examples/encode_batch.rs +0 -0
  128. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/examples/serialization.rs +0 -0
  129. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/src/decoders/bpe.rs +0 -0
  130. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/src/decoders/byte_fallback.rs +0 -0
  131. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/src/decoders/ctc.rs +0 -0
  132. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/src/decoders/fuse.rs +0 -0
  133. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/src/decoders/mod.rs +0 -0
  134. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/src/decoders/sequence.rs +0 -0
  135. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/src/decoders/strip.rs +0 -0
  136. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/src/decoders/wordpiece.rs +0 -0
  137. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/src/lib.rs +0 -0
  138. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/src/models/bpe/mod.rs +0 -0
  139. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/src/models/bpe/model.rs +0 -0
  140. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/src/models/bpe/serialization.rs +0 -0
  141. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/src/models/bpe/word.rs +0 -0
  142. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/src/models/mod.rs +0 -0
  143. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/src/models/unigram/lattice.rs +0 -0
  144. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/src/models/unigram/mod.rs +0 -0
  145. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/src/models/unigram/model.rs +0 -0
  146. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/src/models/unigram/serialization.rs +0 -0
  147. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/src/models/unigram/trainer.rs +0 -0
  148. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/src/models/unigram/trie.rs +0 -0
  149. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/src/models/wordlevel/mod.rs +0 -0
  150. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/src/models/wordlevel/serialization.rs +0 -0
  151. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/src/models/wordlevel/trainer.rs +0 -0
  152. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/src/models/wordpiece/mod.rs +0 -0
  153. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/src/models/wordpiece/serialization.rs +0 -0
  154. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/src/normalizers/bert.rs +0 -0
  155. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/src/normalizers/byte_level.rs +0 -0
  156. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/src/normalizers/mod.rs +0 -0
  157. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/src/normalizers/precompiled.rs +0 -0
  158. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/src/normalizers/prepend.rs +0 -0
  159. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/src/normalizers/replace.rs +0 -0
  160. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/src/normalizers/strip.rs +0 -0
  161. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/src/normalizers/unicode.rs +0 -0
  162. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/src/normalizers/utils.rs +0 -0
  163. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/src/pre_tokenizers/bert.rs +0 -0
  164. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/src/pre_tokenizers/byte_level.rs +0 -0
  165. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/src/pre_tokenizers/delimiter.rs +0 -0
  166. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/src/pre_tokenizers/digits.rs +0 -0
  167. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/src/pre_tokenizers/fixed_length.rs +0 -0
  168. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/src/pre_tokenizers/metaspace.rs +0 -0
  169. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/src/pre_tokenizers/mod.rs +0 -0
  170. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/src/pre_tokenizers/punctuation.rs +0 -0
  171. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/src/pre_tokenizers/sequence.rs +0 -0
  172. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/src/pre_tokenizers/split.rs +0 -0
  173. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/src/pre_tokenizers/unicode_scripts/mod.rs +0 -0
  174. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/src/pre_tokenizers/unicode_scripts/pre_tokenizer.rs +0 -0
  175. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/src/pre_tokenizers/unicode_scripts/scripts.rs +0 -0
  176. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/src/pre_tokenizers/whitespace.rs +0 -0
  177. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/src/processors/bert.rs +0 -0
  178. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/src/processors/mod.rs +0 -0
  179. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/src/processors/roberta.rs +0 -0
  180. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/src/processors/sequence.rs +0 -0
  181. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/src/processors/template.rs +0 -0
  182. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/src/tokenizer/added_vocabulary.rs +0 -0
  183. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/src/tokenizer/encoding.rs +0 -0
  184. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/src/tokenizer/mod.rs +0 -0
  185. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/src/tokenizer/normalizer.rs +0 -0
  186. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/src/tokenizer/pattern.rs +0 -0
  187. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/src/tokenizer/pre_tokenizer.rs +0 -0
  188. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/src/tokenizer/serialization.rs +0 -0
  189. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/src/utils/cache.rs +0 -0
  190. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/src/utils/fancy.rs +0 -0
  191. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/src/utils/from_pretrained.rs +0 -0
  192. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/src/utils/iter.rs +0 -0
  193. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/src/utils/mod.rs +0 -0
  194. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/src/utils/onig.rs +0 -0
  195. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/src/utils/padding.rs +0 -0
  196. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/src/utils/parallelism.rs +0 -0
  197. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/src/utils/progress.rs +0 -0
  198. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/src/utils/truncation.rs +0 -0
  199. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/tests/added_tokens.rs +0 -0
  200. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/tests/common/mod.rs +0 -0
  201. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/tests/documentation.rs +0 -0
  202. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/tests/from_pretrained.rs +0 -0
  203. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/tests/offsets.rs +0 -0
  204. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/tests/serialization.rs +0 -0
  205. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/tests/stream.rs +0 -0
  206. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/tests/training.rs +0 -0
  207. {tokenizers-0.21.2 → tokenizers-0.21.4.dev0}/tokenizers/tests/unigram.rs +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: tokenizers
3
- Version: 0.21.2
3
+ Version: 0.21.4.dev0
4
4
  Classifier: Development Status :: 5 - Production/Stable
5
5
  Classifier: Intended Audience :: Developers
6
6
  Classifier: Intended Audience :: Education
@@ -27,9 +27,9 @@ dependencies = [
27
27
 
28
28
  [[package]]
29
29
  name = "anstream"
30
- version = "0.6.18"
30
+ version = "0.6.19"
31
31
  source = "registry+https://github.com/rust-lang/crates.io-index"
32
- checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b"
32
+ checksum = "301af1932e46185686725e0fad2f8f2aa7da69dd70bf6ecc44d6b703844a3933"
33
33
  dependencies = [
34
34
  "anstyle",
35
35
  "anstyle-parse",
@@ -42,44 +42,44 @@ dependencies = [
42
42
 
43
43
  [[package]]
44
44
  name = "anstyle"
45
- version = "1.0.10"
45
+ version = "1.0.11"
46
46
  source = "registry+https://github.com/rust-lang/crates.io-index"
47
- checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9"
47
+ checksum = "862ed96ca487e809f1c8e5a8447f6ee2cf102f846893800b20cebdf541fc6bbd"
48
48
 
49
49
  [[package]]
50
50
  name = "anstyle-parse"
51
- version = "0.2.6"
51
+ version = "0.2.7"
52
52
  source = "registry+https://github.com/rust-lang/crates.io-index"
53
- checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9"
53
+ checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2"
54
54
  dependencies = [
55
55
  "utf8parse",
56
56
  ]
57
57
 
58
58
  [[package]]
59
59
  name = "anstyle-query"
60
- version = "1.1.2"
60
+ version = "1.1.3"
61
61
  source = "registry+https://github.com/rust-lang/crates.io-index"
62
- checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c"
62
+ checksum = "6c8bdeb6047d8983be085bab0ba1472e6dc604e7041dbf6fcd5e71523014fae9"
63
63
  dependencies = [
64
- "windows-sys",
64
+ "windows-sys 0.59.0",
65
65
  ]
66
66
 
67
67
  [[package]]
68
68
  name = "anstyle-wincon"
69
- version = "3.0.8"
69
+ version = "3.0.9"
70
70
  source = "registry+https://github.com/rust-lang/crates.io-index"
71
- checksum = "6680de5231bd6ee4c6191b8a1325daa282b415391ec9d3a37bd34f2060dc73fa"
71
+ checksum = "403f75924867bb1033c59fbf0797484329750cfbe3c4325cd33127941fabc882"
72
72
  dependencies = [
73
73
  "anstyle",
74
74
  "once_cell_polyfill",
75
- "windows-sys",
75
+ "windows-sys 0.59.0",
76
76
  ]
77
77
 
78
78
  [[package]]
79
79
  name = "autocfg"
80
- version = "1.4.0"
80
+ version = "1.5.0"
81
81
  source = "registry+https://github.com/rust-lang/crates.io-index"
82
- checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26"
82
+ checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8"
83
83
 
84
84
  [[package]]
85
85
  name = "base64"
@@ -95,9 +95,9 @@ checksum = "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967"
95
95
 
96
96
  [[package]]
97
97
  name = "bumpalo"
98
- version = "3.17.0"
98
+ version = "3.19.0"
99
99
  source = "registry+https://github.com/rust-lang/crates.io-index"
100
- checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf"
100
+ checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43"
101
101
 
102
102
  [[package]]
103
103
  name = "castaway"
@@ -110,24 +110,24 @@ dependencies = [
110
110
 
111
111
  [[package]]
112
112
  name = "cc"
113
- version = "1.2.24"
113
+ version = "1.2.27"
114
114
  source = "registry+https://github.com/rust-lang/crates.io-index"
115
- checksum = "16595d3be041c03b09d08d0858631facccee9221e579704070e6e9e4915d3bc7"
115
+ checksum = "d487aa071b5f64da6f19a3e848e3578944b726ee5a4854b82172f02aa876bfdc"
116
116
  dependencies = [
117
117
  "shlex",
118
118
  ]
119
119
 
120
120
  [[package]]
121
121
  name = "cfg-if"
122
- version = "1.0.0"
122
+ version = "1.0.1"
123
123
  source = "registry+https://github.com/rust-lang/crates.io-index"
124
- checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
124
+ checksum = "9555578bc9e57714c812a1f84e4fc5b4d21fcb063490c624de019f7464c91268"
125
125
 
126
126
  [[package]]
127
127
  name = "colorchoice"
128
- version = "1.0.3"
128
+ version = "1.0.4"
129
129
  source = "registry+https://github.com/rust-lang/crates.io-index"
130
- checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990"
130
+ checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75"
131
131
 
132
132
  [[package]]
133
133
  name = "compact_str"
@@ -154,7 +154,7 @@ dependencies = [
154
154
  "libc",
155
155
  "once_cell",
156
156
  "unicode-width",
157
- "windows-sys",
157
+ "windows-sys 0.59.0",
158
158
  ]
159
159
 
160
160
  [[package]]
@@ -294,12 +294,12 @@ dependencies = [
294
294
 
295
295
  [[package]]
296
296
  name = "errno"
297
- version = "0.3.12"
297
+ version = "0.3.13"
298
298
  source = "registry+https://github.com/rust-lang/crates.io-index"
299
- checksum = "cea14ef9355e3beab063703aa9dab15afd25f0667c341310c1e5274bb1d0da18"
299
+ checksum = "778e2ac28f6c47af28e4907f13ffd1e1ddbd400980a9abd7c8df189bf578a5ad"
300
300
  dependencies = [
301
301
  "libc",
302
- "windows-sys",
302
+ "windows-sys 0.60.2",
303
303
  ]
304
304
 
305
305
  [[package]]
@@ -389,9 +389,9 @@ checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c"
389
389
 
390
390
  [[package]]
391
391
  name = "jiff"
392
- version = "0.2.14"
392
+ version = "0.2.15"
393
393
  source = "registry+https://github.com/rust-lang/crates.io-index"
394
- checksum = "a194df1107f33c79f4f93d02c80798520551949d59dfad22b6157048a88cca93"
394
+ checksum = "be1f93b8b1eb69c77f24bbb0afdf66f54b632ee39af40ca21c4365a1d7347e49"
395
395
  dependencies = [
396
396
  "jiff-static",
397
397
  "log",
@@ -402,9 +402,9 @@ dependencies = [
402
402
 
403
403
  [[package]]
404
404
  name = "jiff-static"
405
- version = "0.2.14"
405
+ version = "0.2.15"
406
406
  source = "registry+https://github.com/rust-lang/crates.io-index"
407
- checksum = "6c6e1db7ed32c6c71b759497fae34bf7933636f75a251b9e736555da426f6442"
407
+ checksum = "03343451ff899767262ec32146f6d559dd759fdadf42ff0e227c7c48f72594b4"
408
408
  dependencies = [
409
409
  "proc-macro2",
410
410
  "quote",
@@ -423,9 +423,9 @@ dependencies = [
423
423
 
424
424
  [[package]]
425
425
  name = "libc"
426
- version = "0.2.172"
426
+ version = "0.2.174"
427
427
  source = "registry+https://github.com/rust-lang/crates.io-index"
428
- checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa"
428
+ checksum = "1171693293099992e19cddea4e8b849964e9846f4acee11b3948bcc337be8776"
429
429
 
430
430
  [[package]]
431
431
  name = "linux-raw-sys"
@@ -441,9 +441,9 @@ checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94"
441
441
 
442
442
  [[package]]
443
443
  name = "macro_rules_attribute"
444
- version = "0.2.0"
444
+ version = "0.2.2"
445
445
  source = "registry+https://github.com/rust-lang/crates.io-index"
446
- checksum = "8a82271f7bc033d84bbca59a3ce3e4159938cb08a9c3aebbe54d215131518a13"
446
+ checksum = "65049d7923698040cd0b1ddcced9b0eb14dd22c5f86ae59c3740eab64a676520"
447
447
  dependencies = [
448
448
  "macro_rules_attribute-proc_macro",
449
449
  "paste",
@@ -451,9 +451,9 @@ dependencies = [
451
451
 
452
452
  [[package]]
453
453
  name = "macro_rules_attribute-proc_macro"
454
- version = "0.2.0"
454
+ version = "0.2.2"
455
455
  source = "registry+https://github.com/rust-lang/crates.io-index"
456
- checksum = "b8dd856d451cc0da70e2ef2ce95a18e39a93b7558bedf10201ad28503f918568"
456
+ checksum = "670fdfda89751bc4a84ac13eaa63e205cf0fd22b4c9a5fbfa085b63c1f1d3a30"
457
457
 
458
458
  [[package]]
459
459
  name = "matrixmultiply"
@@ -467,9 +467,9 @@ dependencies = [
467
467
 
468
468
  [[package]]
469
469
  name = "memchr"
470
- version = "2.7.4"
470
+ version = "2.7.5"
471
471
  source = "registry+https://github.com/rust-lang/crates.io-index"
472
- checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3"
472
+ checksum = "32a282da65faaf38286cf3be983213fcf1d2e2a58700e808f83f4ea9a4804bc0"
473
473
 
474
474
  [[package]]
475
475
  name = "memoffset"
@@ -629,9 +629,9 @@ checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c"
629
629
 
630
630
  [[package]]
631
631
  name = "portable-atomic"
632
- version = "1.11.0"
632
+ version = "1.11.1"
633
633
  source = "registry+https://github.com/rust-lang/crates.io-index"
634
- checksum = "350e9b48cbc6b0e028b0473b114454c6316e57336ee184ceab6e53f72c178b3e"
634
+ checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483"
635
635
 
636
636
  [[package]]
637
637
  name = "portable-atomic-util"
@@ -733,9 +733,9 @@ dependencies = [
733
733
 
734
734
  [[package]]
735
735
  name = "r-efi"
736
- version = "5.2.0"
736
+ version = "5.3.0"
737
737
  source = "registry+https://github.com/rust-lang/crates.io-index"
738
- checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5"
738
+ checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f"
739
739
 
740
740
  [[package]]
741
741
  name = "rand"
@@ -848,7 +848,7 @@ dependencies = [
848
848
  "errno",
849
849
  "libc",
850
850
  "linux-raw-sys",
851
- "windows-sys",
851
+ "windows-sys 0.59.0",
852
852
  ]
853
853
 
854
854
  [[package]]
@@ -903,9 +903,9 @@ checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
903
903
 
904
904
  [[package]]
905
905
  name = "smallvec"
906
- version = "1.15.0"
906
+ version = "1.15.1"
907
907
  source = "registry+https://github.com/rust-lang/crates.io-index"
908
- checksum = "8917285742e9f3e1683f0a9c4e6b57960b7314d0b08d30d1ecd426713ee2eee9"
908
+ checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03"
909
909
 
910
910
  [[package]]
911
911
  name = "spm_precompiled"
@@ -933,9 +933,9 @@ checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
933
933
 
934
934
  [[package]]
935
935
  name = "syn"
936
- version = "2.0.101"
936
+ version = "2.0.104"
937
937
  source = "registry+https://github.com/rust-lang/crates.io-index"
938
- checksum = "8ce2b7fc941b3a24138a0a7cf8e858bfc6a992e7978a068a5c760deb0ed43caf"
938
+ checksum = "17b6f705963418cdb9927482fa304bc562ece2fdd4f616084c50b7023b435a40"
939
939
  dependencies = [
940
940
  "proc-macro2",
941
941
  "quote",
@@ -958,7 +958,7 @@ dependencies = [
958
958
  "getrandom",
959
959
  "once_cell",
960
960
  "rustix",
961
- "windows-sys",
961
+ "windows-sys 0.59.0",
962
962
  ]
963
963
 
964
964
  [[package]]
@@ -983,7 +983,7 @@ dependencies = [
983
983
 
984
984
  [[package]]
985
985
  name = "tokenizers"
986
- version = "0.21.2"
986
+ version = "0.21.4-dev.0"
987
987
  dependencies = [
988
988
  "ahash",
989
989
  "aho-corasick",
@@ -1015,7 +1015,7 @@ dependencies = [
1015
1015
 
1016
1016
  [[package]]
1017
1017
  name = "tokenizers-python"
1018
- version = "0.21.2"
1018
+ version = "0.21.4-dev.0"
1019
1019
  dependencies = [
1020
1020
  "ahash",
1021
1021
  "env_logger",
@@ -1054,9 +1054,9 @@ checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493"
1054
1054
 
1055
1055
  [[package]]
1056
1056
  name = "unicode-width"
1057
- version = "0.2.0"
1057
+ version = "0.2.1"
1058
1058
  source = "registry+https://github.com/rust-lang/crates.io-index"
1059
- checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd"
1059
+ checksum = "4a1a07cc7db3810833284e8d372ccdc6da29741639ecc70c9ec107df0fa6154c"
1060
1060
 
1061
1061
  [[package]]
1062
1062
  name = "unicode_categories"
@@ -1164,7 +1164,16 @@ version = "0.59.0"
1164
1164
  source = "registry+https://github.com/rust-lang/crates.io-index"
1165
1165
  checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b"
1166
1166
  dependencies = [
1167
- "windows-targets",
1167
+ "windows-targets 0.52.6",
1168
+ ]
1169
+
1170
+ [[package]]
1171
+ name = "windows-sys"
1172
+ version = "0.60.2"
1173
+ source = "registry+https://github.com/rust-lang/crates.io-index"
1174
+ checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb"
1175
+ dependencies = [
1176
+ "windows-targets 0.53.2",
1168
1177
  ]
1169
1178
 
1170
1179
  [[package]]
@@ -1173,14 +1182,30 @@ version = "0.52.6"
1173
1182
  source = "registry+https://github.com/rust-lang/crates.io-index"
1174
1183
  checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"
1175
1184
  dependencies = [
1176
- "windows_aarch64_gnullvm",
1177
- "windows_aarch64_msvc",
1178
- "windows_i686_gnu",
1179
- "windows_i686_gnullvm",
1180
- "windows_i686_msvc",
1181
- "windows_x86_64_gnu",
1182
- "windows_x86_64_gnullvm",
1183
- "windows_x86_64_msvc",
1185
+ "windows_aarch64_gnullvm 0.52.6",
1186
+ "windows_aarch64_msvc 0.52.6",
1187
+ "windows_i686_gnu 0.52.6",
1188
+ "windows_i686_gnullvm 0.52.6",
1189
+ "windows_i686_msvc 0.52.6",
1190
+ "windows_x86_64_gnu 0.52.6",
1191
+ "windows_x86_64_gnullvm 0.52.6",
1192
+ "windows_x86_64_msvc 0.52.6",
1193
+ ]
1194
+
1195
+ [[package]]
1196
+ name = "windows-targets"
1197
+ version = "0.53.2"
1198
+ source = "registry+https://github.com/rust-lang/crates.io-index"
1199
+ checksum = "c66f69fcc9ce11da9966ddb31a40968cad001c5bedeb5c2b82ede4253ab48aef"
1200
+ dependencies = [
1201
+ "windows_aarch64_gnullvm 0.53.0",
1202
+ "windows_aarch64_msvc 0.53.0",
1203
+ "windows_i686_gnu 0.53.0",
1204
+ "windows_i686_gnullvm 0.53.0",
1205
+ "windows_i686_msvc 0.53.0",
1206
+ "windows_x86_64_gnu 0.53.0",
1207
+ "windows_x86_64_gnullvm 0.53.0",
1208
+ "windows_x86_64_msvc 0.53.0",
1184
1209
  ]
1185
1210
 
1186
1211
  [[package]]
@@ -1189,48 +1214,96 @@ version = "0.52.6"
1189
1214
  source = "registry+https://github.com/rust-lang/crates.io-index"
1190
1215
  checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
1191
1216
 
1217
+ [[package]]
1218
+ name = "windows_aarch64_gnullvm"
1219
+ version = "0.53.0"
1220
+ source = "registry+https://github.com/rust-lang/crates.io-index"
1221
+ checksum = "86b8d5f90ddd19cb4a147a5fa63ca848db3df085e25fee3cc10b39b6eebae764"
1222
+
1192
1223
  [[package]]
1193
1224
  name = "windows_aarch64_msvc"
1194
1225
  version = "0.52.6"
1195
1226
  source = "registry+https://github.com/rust-lang/crates.io-index"
1196
1227
  checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
1197
1228
 
1229
+ [[package]]
1230
+ name = "windows_aarch64_msvc"
1231
+ version = "0.53.0"
1232
+ source = "registry+https://github.com/rust-lang/crates.io-index"
1233
+ checksum = "c7651a1f62a11b8cbd5e0d42526e55f2c99886c77e007179efff86c2b137e66c"
1234
+
1198
1235
  [[package]]
1199
1236
  name = "windows_i686_gnu"
1200
1237
  version = "0.52.6"
1201
1238
  source = "registry+https://github.com/rust-lang/crates.io-index"
1202
1239
  checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"
1203
1240
 
1241
+ [[package]]
1242
+ name = "windows_i686_gnu"
1243
+ version = "0.53.0"
1244
+ source = "registry+https://github.com/rust-lang/crates.io-index"
1245
+ checksum = "c1dc67659d35f387f5f6c479dc4e28f1d4bb90ddd1a5d3da2e5d97b42d6272c3"
1246
+
1204
1247
  [[package]]
1205
1248
  name = "windows_i686_gnullvm"
1206
1249
  version = "0.52.6"
1207
1250
  source = "registry+https://github.com/rust-lang/crates.io-index"
1208
1251
  checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
1209
1252
 
1253
+ [[package]]
1254
+ name = "windows_i686_gnullvm"
1255
+ version = "0.53.0"
1256
+ source = "registry+https://github.com/rust-lang/crates.io-index"
1257
+ checksum = "9ce6ccbdedbf6d6354471319e781c0dfef054c81fbc7cf83f338a4296c0cae11"
1258
+
1210
1259
  [[package]]
1211
1260
  name = "windows_i686_msvc"
1212
1261
  version = "0.52.6"
1213
1262
  source = "registry+https://github.com/rust-lang/crates.io-index"
1214
1263
  checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
1215
1264
 
1265
+ [[package]]
1266
+ name = "windows_i686_msvc"
1267
+ version = "0.53.0"
1268
+ source = "registry+https://github.com/rust-lang/crates.io-index"
1269
+ checksum = "581fee95406bb13382d2f65cd4a908ca7b1e4c2f1917f143ba16efe98a589b5d"
1270
+
1216
1271
  [[package]]
1217
1272
  name = "windows_x86_64_gnu"
1218
1273
  version = "0.52.6"
1219
1274
  source = "registry+https://github.com/rust-lang/crates.io-index"
1220
1275
  checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
1221
1276
 
1277
+ [[package]]
1278
+ name = "windows_x86_64_gnu"
1279
+ version = "0.53.0"
1280
+ source = "registry+https://github.com/rust-lang/crates.io-index"
1281
+ checksum = "2e55b5ac9ea33f2fc1716d1742db15574fd6fc8dadc51caab1c16a3d3b4190ba"
1282
+
1222
1283
  [[package]]
1223
1284
  name = "windows_x86_64_gnullvm"
1224
1285
  version = "0.52.6"
1225
1286
  source = "registry+https://github.com/rust-lang/crates.io-index"
1226
1287
  checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
1227
1288
 
1289
+ [[package]]
1290
+ name = "windows_x86_64_gnullvm"
1291
+ version = "0.53.0"
1292
+ source = "registry+https://github.com/rust-lang/crates.io-index"
1293
+ checksum = "0a6e035dd0599267ce1ee132e51c27dd29437f63325753051e71dd9e42406c57"
1294
+
1228
1295
  [[package]]
1229
1296
  name = "windows_x86_64_msvc"
1230
1297
  version = "0.52.6"
1231
1298
  source = "registry+https://github.com/rust-lang/crates.io-index"
1232
1299
  checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
1233
1300
 
1301
+ [[package]]
1302
+ name = "windows_x86_64_msvc"
1303
+ version = "0.53.0"
1304
+ source = "registry+https://github.com/rust-lang/crates.io-index"
1305
+ checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486"
1306
+
1234
1307
  [[package]]
1235
1308
  name = "wit-bindgen-rt"
1236
1309
  version = "0.39.0"
@@ -1242,18 +1315,18 @@ dependencies = [
1242
1315
 
1243
1316
  [[package]]
1244
1317
  name = "zerocopy"
1245
- version = "0.8.25"
1318
+ version = "0.8.26"
1246
1319
  source = "registry+https://github.com/rust-lang/crates.io-index"
1247
- checksum = "a1702d9583232ddb9174e01bb7c15a2ab8fb1bc6f227aa1233858c351a3ba0cb"
1320
+ checksum = "1039dd0d3c310cf05de012d8a39ff557cb0d23087fd44cad61df08fc31907a2f"
1248
1321
  dependencies = [
1249
1322
  "zerocopy-derive",
1250
1323
  ]
1251
1324
 
1252
1325
  [[package]]
1253
1326
  name = "zerocopy-derive"
1254
- version = "0.8.25"
1327
+ version = "0.8.26"
1255
1328
  source = "registry+https://github.com/rust-lang/crates.io-index"
1256
- checksum = "28a6e20d751156648aa063f3800b706ee209a32c0b4d9f24be3d980b01be55ef"
1329
+ checksum = "9ecf5b4cc5364572d7f4c329661bcc82724222973f2cab6f050a4e5c22f75181"
1257
1330
  dependencies = [
1258
1331
  "proc-macro2",
1259
1332
  "quote",
@@ -1,6 +1,6 @@
1
1
  [package]
2
2
  name = "tokenizers-python"
3
- version = "0.21.2"
3
+ version = "0.21.4-dev.0"
4
4
  authors = ["Anthony MOI <m.anthony.moi@gmail.com>"]
5
5
  edition = "2021"
6
6
 
@@ -107,8 +107,7 @@ impl PyDecoder {
107
107
  fn __getstate__(&self, py: Python) -> PyResult<PyObject> {
108
108
  let data = serde_json::to_string(&self.decoder).map_err(|e| {
109
109
  exceptions::PyException::new_err(format!(
110
- "Error while attempting to pickle Decoder: {}",
111
- e
110
+ "Error while attempting to pickle Decoder: {e}"
112
111
  ))
113
112
  })?;
114
113
  Ok(PyBytes::new(py, data.as_bytes()).into())
@@ -119,8 +118,7 @@ impl PyDecoder {
119
118
  Ok(s) => {
120
119
  self.decoder = serde_json::from_slice(s).map_err(|e| {
121
120
  exceptions::PyException::new_err(format!(
122
- "Error while attempting to unpickle Decoder: {}",
123
- e
121
+ "Error while attempting to unpickle Decoder: {e}"
124
122
  ))
125
123
  })?;
126
124
  Ok(())
@@ -33,8 +33,7 @@ impl PyEncoding {
33
33
  fn __getstate__(&self, py: Python) -> PyResult<PyObject> {
34
34
  let data = serde_json::to_string(&self.encoding).map_err(|e| {
35
35
  exceptions::PyException::new_err(format!(
36
- "Error while attempting to pickle Encoding: {}",
37
- e
36
+ "Error while attempting to pickle Encoding: {e}"
38
37
  ))
39
38
  })?;
40
39
  Ok(PyBytes::new(py, data.as_bytes()).into())
@@ -45,8 +44,7 @@ impl PyEncoding {
45
44
  Ok(s) => {
46
45
  self.encoding = serde_json::from_slice(s).map_err(|e| {
47
46
  exceptions::PyException::new_err(format!(
48
- "Error while attempting to unpickle Encoding: {}",
49
- e
47
+ "Error while attempting to unpickle Encoding: {e}"
50
48
  ))
51
49
  })?;
52
50
  Ok(())
@@ -407,9 +405,8 @@ impl PyEncoding {
407
405
  "left" => Ok(PaddingDirection::Left),
408
406
  "right" => Ok(PaddingDirection::Right),
409
407
  other => Err(PyError(format!(
410
- "Unknown `direction`: `{}`. Use \
411
- one of `left` or `right`",
412
- other
408
+ "Unknown `direction`: `{other}`. Use \
409
+ one of `left` or `right`"
413
410
  ))
414
411
  .into_pyerr::<exceptions::PyValueError>()),
415
412
  }?;
@@ -417,7 +414,7 @@ impl PyEncoding {
417
414
  "pad_id" => pad_id = value.extract()?,
418
415
  "pad_type_id" => pad_type_id = value.extract()?,
419
416
  "pad_token" => pad_token = value.extract()?,
420
- _ => println!("Ignored unknown kwarg option {}", key),
417
+ _ => println!("Ignored unknown kwarg option {key}"),
421
418
  }
422
419
  }
423
420
  }
@@ -446,11 +443,10 @@ impl PyEncoding {
446
443
  let tdir = match direction {
447
444
  "left" => Ok(TruncationDirection::Left),
448
445
  "right" => Ok(TruncationDirection::Right),
449
- _ => Err(PyError(format!(
450
- "Invalid truncation direction value : {}",
451
- direction
452
- ))
453
- .into_pyerr::<exceptions::PyValueError>()),
446
+ _ => Err(
447
+ PyError(format!("Invalid truncation direction value : {direction}"))
448
+ .into_pyerr::<exceptions::PyValueError>(),
449
+ ),
454
450
  }?;
455
451
 
456
452
  self.encoding.truncate(max_length, stride, tdir);
@@ -13,7 +13,7 @@ impl PyError {
13
13
  PyError(String::from(s))
14
14
  }
15
15
  pub fn into_pyerr<T: PyTypeInfo>(self) -> PyErr {
16
- PyErr::new::<T, _>(format!("{}", self))
16
+ PyErr::new::<T, _>(format!("{self}"))
17
17
  }
18
18
  }
19
19
  impl Display for PyError {
@@ -26,7 +26,7 @@ impl std::error::Error for PyError {}
26
26
  pub struct ToPyResult<T>(pub Result<T>);
27
27
  impl<T> From<ToPyResult<T>> for PyResult<T> {
28
28
  fn from(v: ToPyResult<T>) -> Self {
29
- v.0.map_err(|e| exceptions::PyException::new_err(format!("{}", e)))
29
+ v.0.map_err(|e| exceptions::PyException::new_err(format!("{e}")))
30
30
  }
31
31
  }
32
32
  impl<T> ToPyResult<T> {
@@ -37,6 +37,6 @@ impl<T> ToPyResult<T> {
37
37
 
38
38
  pub(crate) fn deprecation_warning(py: Python<'_>, version: &str, message: &str) -> PyResult<()> {
39
39
  let deprecation_warning = py.import("builtins")?.getattr("DeprecationWarning")?;
40
- let full_message = format!("Deprecated in {}: {}", version, message);
40
+ let full_message = format!("Deprecated in {version}: {message}");
41
41
  pyo3::PyErr::warn(py, &deprecation_warning, &CString::new(full_message)?, 0)
42
42
  }
@@ -38,8 +38,7 @@ extern "C" fn child_after_fork() {
38
38
  eprintln!("To disable this warning, you can either:");
39
39
  eprintln!(
40
40
  "\t- Avoid using `tokenizers` before the fork if possible\n\
41
- \t- Explicitly set the environment variable {}=(true | false)",
42
- ENV_VARIABLE
41
+ \t- Explicitly set the environment variable {ENV_VARIABLE}=(true | false)"
43
42
  );
44
43
  set_parallelism(false);
45
44
  }