tokenizers 0.22.2__tar.gz → 0.22.2rc0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (211) hide show
  1. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/PKG-INFO +1 -1
  2. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/Cargo.lock +2 -2
  3. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/Cargo.toml +1 -1
  4. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/Cargo.toml +1 -1
  5. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/.cargo/config.toml +0 -0
  6. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/.gitignore +0 -0
  7. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/CHANGELOG.md +0 -0
  8. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/MANIFEST.in +0 -0
  9. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/Makefile +0 -0
  10. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/README.md +0 -0
  11. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/benches/test_tiktoken.py +0 -0
  12. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/conftest.py +0 -0
  13. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/docs/pyo3.md +0 -0
  14. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/examples/custom_components.py +0 -0
  15. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/examples/example.py +0 -0
  16. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/examples/train_bert_wordpiece.py +0 -0
  17. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/examples/train_bytelevel_bpe.py +0 -0
  18. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/examples/train_with_datasets.py +0 -0
  19. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/examples/using_the_visualizer.ipynb +0 -0
  20. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/py_src/tokenizers/__init__.py +0 -0
  21. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/py_src/tokenizers/__init__.pyi +0 -0
  22. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/py_src/tokenizers/decoders/__init__.py +0 -0
  23. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/py_src/tokenizers/decoders/__init__.pyi +0 -0
  24. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/py_src/tokenizers/implementations/__init__.py +0 -0
  25. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/py_src/tokenizers/implementations/base_tokenizer.py +0 -0
  26. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/py_src/tokenizers/implementations/bert_wordpiece.py +0 -0
  27. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/py_src/tokenizers/implementations/byte_level_bpe.py +0 -0
  28. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/py_src/tokenizers/implementations/char_level_bpe.py +0 -0
  29. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/py_src/tokenizers/implementations/sentencepiece_bpe.py +0 -0
  30. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/py_src/tokenizers/implementations/sentencepiece_unigram.py +0 -0
  31. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/py_src/tokenizers/models/__init__.py +0 -0
  32. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/py_src/tokenizers/models/__init__.pyi +0 -0
  33. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/py_src/tokenizers/normalizers/__init__.py +0 -0
  34. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/py_src/tokenizers/normalizers/__init__.pyi +0 -0
  35. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/py_src/tokenizers/pre_tokenizers/__init__.py +0 -0
  36. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/py_src/tokenizers/pre_tokenizers/__init__.pyi +0 -0
  37. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/py_src/tokenizers/processors/__init__.py +0 -0
  38. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/py_src/tokenizers/processors/__init__.pyi +0 -0
  39. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/py_src/tokenizers/tokenizers.pyi +0 -0
  40. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/py_src/tokenizers/tools/__init__.py +0 -0
  41. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/py_src/tokenizers/tools/visualizer-styles.css +0 -0
  42. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/py_src/tokenizers/tools/visualizer.py +0 -0
  43. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/py_src/tokenizers/trainers/__init__.py +0 -0
  44. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/py_src/tokenizers/trainers/__init__.pyi +0 -0
  45. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/rust-toolchain +0 -0
  46. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/scripts/convert.py +0 -0
  47. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/scripts/sentencepiece_extractor.py +0 -0
  48. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/scripts/spm_parity_check.py +0 -0
  49. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/setup.cfg +0 -0
  50. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/src/decoders.rs +0 -0
  51. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/src/encoding.rs +0 -0
  52. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/src/error.rs +0 -0
  53. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/src/lib.rs +0 -0
  54. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/src/models.rs +0 -0
  55. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/src/normalizers.rs +0 -0
  56. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/src/pre_tokenizers.rs +0 -0
  57. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/src/processors.rs +0 -0
  58. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/src/token.rs +0 -0
  59. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/src/tokenizer.rs +0 -0
  60. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/src/trainers.rs +0 -0
  61. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/src/utils/iterators.rs +0 -0
  62. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/src/utils/mod.rs +0 -0
  63. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/src/utils/normalization.rs +0 -0
  64. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/src/utils/pretokenization.rs +0 -0
  65. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/src/utils/regex.rs +0 -0
  66. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/src/utils/serde_pyo3.rs +0 -0
  67. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/stub.py +0 -0
  68. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/test.txt +0 -0
  69. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/tests/__init__.py +0 -0
  70. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/tests/bindings/__init__.py +0 -0
  71. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/tests/bindings/test_decoders.py +0 -0
  72. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/tests/bindings/test_encoding.py +0 -0
  73. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/tests/bindings/test_models.py +0 -0
  74. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/tests/bindings/test_normalizers.py +0 -0
  75. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/tests/bindings/test_pre_tokenizers.py +0 -0
  76. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/tests/bindings/test_processors.py +0 -0
  77. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/tests/bindings/test_tokenizer.py +0 -0
  78. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/tests/bindings/test_trainers.py +0 -0
  79. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/tests/documentation/__init__.py +0 -0
  80. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/tests/documentation/test_pipeline.py +0 -0
  81. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/tests/documentation/test_quicktour.py +0 -0
  82. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/tests/documentation/test_tutorial_train_from_iterators.py +0 -0
  83. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/tests/implementations/__init__.py +0 -0
  84. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/tests/implementations/test_base_tokenizer.py +0 -0
  85. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/tests/implementations/test_bert_wordpiece.py +0 -0
  86. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/tests/implementations/test_byte_level_bpe.py +0 -0
  87. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/tests/implementations/test_char_bpe.py +0 -0
  88. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/tests/implementations/test_sentencepiece.py +0 -0
  89. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/tests/test_serialization.py +0 -0
  90. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/tests/utils.py +0 -0
  91. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/py_src/tokenizers/__init__.py +0 -0
  92. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/py_src/tokenizers/__init__.pyi +0 -0
  93. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/py_src/tokenizers/decoders/__init__.py +0 -0
  94. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/py_src/tokenizers/decoders/__init__.pyi +0 -0
  95. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/py_src/tokenizers/implementations/__init__.py +0 -0
  96. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/py_src/tokenizers/implementations/base_tokenizer.py +0 -0
  97. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/py_src/tokenizers/implementations/bert_wordpiece.py +0 -0
  98. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/py_src/tokenizers/implementations/byte_level_bpe.py +0 -0
  99. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/py_src/tokenizers/implementations/char_level_bpe.py +0 -0
  100. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/py_src/tokenizers/implementations/sentencepiece_bpe.py +0 -0
  101. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/py_src/tokenizers/implementations/sentencepiece_unigram.py +0 -0
  102. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/py_src/tokenizers/models/__init__.py +0 -0
  103. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/py_src/tokenizers/models/__init__.pyi +0 -0
  104. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/py_src/tokenizers/normalizers/__init__.py +0 -0
  105. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/py_src/tokenizers/normalizers/__init__.pyi +0 -0
  106. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/py_src/tokenizers/pre_tokenizers/__init__.py +0 -0
  107. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/py_src/tokenizers/pre_tokenizers/__init__.pyi +0 -0
  108. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/py_src/tokenizers/processors/__init__.py +0 -0
  109. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/py_src/tokenizers/processors/__init__.pyi +0 -0
  110. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/py_src/tokenizers/tokenizers.pyi +0 -0
  111. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/py_src/tokenizers/tools/__init__.py +0 -0
  112. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/py_src/tokenizers/tools/visualizer-styles.css +0 -0
  113. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/py_src/tokenizers/tools/visualizer.py +0 -0
  114. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/py_src/tokenizers/trainers/__init__.py +0 -0
  115. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/py_src/tokenizers/trainers/__init__.pyi +0 -0
  116. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/pyproject.toml +0 -0
  117. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/CHANGELOG.md +0 -0
  118. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/LICENSE +0 -0
  119. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/Makefile +0 -0
  120. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/README.md +0 -0
  121. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/README.tpl +0 -0
  122. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/benches/added_vocab_deserialize.rs +0 -0
  123. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/benches/bert_benchmark.rs +0 -0
  124. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/benches/bpe_benchmark.rs +0 -0
  125. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/benches/common/mod.rs +0 -0
  126. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/benches/layout_benchmark.rs +0 -0
  127. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/benches/llama3_benchmark.rs +0 -0
  128. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/benches/unigram_benchmark.rs +0 -0
  129. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/examples/encode_batch.rs +0 -0
  130. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/examples/serialization.rs +0 -0
  131. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/decoders/bpe.rs +0 -0
  132. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/decoders/byte_fallback.rs +0 -0
  133. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/decoders/ctc.rs +0 -0
  134. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/decoders/fuse.rs +0 -0
  135. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/decoders/mod.rs +0 -0
  136. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/decoders/sequence.rs +0 -0
  137. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/decoders/strip.rs +0 -0
  138. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/decoders/wordpiece.rs +0 -0
  139. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/lib.rs +0 -0
  140. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/models/bpe/mod.rs +0 -0
  141. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/models/bpe/model.rs +0 -0
  142. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/models/bpe/serialization.rs +0 -0
  143. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/models/bpe/trainer.rs +0 -0
  144. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/models/bpe/word.rs +0 -0
  145. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/models/mod.rs +0 -0
  146. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/models/unigram/lattice.rs +0 -0
  147. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/models/unigram/mod.rs +0 -0
  148. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/models/unigram/model.rs +0 -0
  149. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/models/unigram/serialization.rs +0 -0
  150. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/models/unigram/trainer.rs +0 -0
  151. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/models/unigram/trie.rs +0 -0
  152. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/models/wordlevel/mod.rs +0 -0
  153. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/models/wordlevel/serialization.rs +0 -0
  154. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/models/wordlevel/trainer.rs +0 -0
  155. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/models/wordpiece/mod.rs +0 -0
  156. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/models/wordpiece/serialization.rs +0 -0
  157. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/models/wordpiece/trainer.rs +0 -0
  158. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/normalizers/bert.rs +0 -0
  159. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/normalizers/byte_level.rs +0 -0
  160. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/normalizers/mod.rs +0 -0
  161. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/normalizers/precompiled.rs +0 -0
  162. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/normalizers/prepend.rs +0 -0
  163. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/normalizers/replace.rs +0 -0
  164. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/normalizers/strip.rs +0 -0
  165. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/normalizers/unicode.rs +0 -0
  166. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/normalizers/utils.rs +0 -0
  167. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/pre_tokenizers/bert.rs +0 -0
  168. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/pre_tokenizers/byte_level.rs +0 -0
  169. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/pre_tokenizers/delimiter.rs +0 -0
  170. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/pre_tokenizers/digits.rs +0 -0
  171. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/pre_tokenizers/fixed_length.rs +0 -0
  172. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/pre_tokenizers/metaspace.rs +0 -0
  173. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/pre_tokenizers/mod.rs +0 -0
  174. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/pre_tokenizers/punctuation.rs +0 -0
  175. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/pre_tokenizers/sequence.rs +0 -0
  176. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/pre_tokenizers/split.rs +0 -0
  177. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/pre_tokenizers/unicode_scripts/mod.rs +0 -0
  178. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/pre_tokenizers/unicode_scripts/pre_tokenizer.rs +0 -0
  179. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/pre_tokenizers/unicode_scripts/scripts.rs +0 -0
  180. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/pre_tokenizers/whitespace.rs +0 -0
  181. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/processors/bert.rs +0 -0
  182. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/processors/mod.rs +0 -0
  183. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/processors/roberta.rs +0 -0
  184. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/processors/sequence.rs +0 -0
  185. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/processors/template.rs +0 -0
  186. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/tokenizer/added_vocabulary.rs +0 -0
  187. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/tokenizer/encoding.rs +0 -0
  188. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/tokenizer/mod.rs +0 -0
  189. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/tokenizer/normalizer.rs +0 -0
  190. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/tokenizer/pattern.rs +0 -0
  191. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/tokenizer/pre_tokenizer.rs +0 -0
  192. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/tokenizer/serialization.rs +0 -0
  193. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/utils/cache.rs +0 -0
  194. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/utils/fancy.rs +0 -0
  195. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/utils/from_pretrained.rs +0 -0
  196. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/utils/iter.rs +0 -0
  197. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/utils/mod.rs +0 -0
  198. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/utils/onig.rs +0 -0
  199. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/utils/padding.rs +0 -0
  200. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/utils/parallelism.rs +0 -0
  201. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/utils/progress.rs +0 -0
  202. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/utils/truncation.rs +0 -0
  203. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/tests/added_tokens.rs +0 -0
  204. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/tests/common/mod.rs +0 -0
  205. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/tests/documentation.rs +0 -0
  206. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/tests/from_pretrained.rs +0 -0
  207. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/tests/offsets.rs +0 -0
  208. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/tests/serialization.rs +0 -0
  209. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/tests/stream.rs +0 -0
  210. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/tests/training.rs +0 -0
  211. {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/tests/unigram.rs +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: tokenizers
3
- Version: 0.22.2
3
+ Version: 0.22.2rc0
4
4
  Classifier: Development Status :: 5 - Production/Stable
5
5
  Classifier: Intended Audience :: Developers
6
6
  Classifier: Intended Audience :: Education
@@ -1138,7 +1138,7 @@ dependencies = [
1138
1138
 
1139
1139
  [[package]]
1140
1140
  name = "tokenizers"
1141
- version = "0.22.2"
1141
+ version = "0.22.2-rc0"
1142
1142
  dependencies = [
1143
1143
  "ahash",
1144
1144
  "aho-corasick",
@@ -1170,7 +1170,7 @@ dependencies = [
1170
1170
 
1171
1171
  [[package]]
1172
1172
  name = "tokenizers-python"
1173
- version = "0.22.2"
1173
+ version = "0.22.2-rc0"
1174
1174
  dependencies = [
1175
1175
  "ahash",
1176
1176
  "env_logger",
@@ -1,6 +1,6 @@
1
1
  [package]
2
2
  name = "tokenizers-python"
3
- version = "0.22.2"
3
+ version = "0.22.2-rc0"
4
4
  authors = ["Anthony MOI <m.anthony.moi@gmail.com>"]
5
5
  edition = "2021"
6
6
  readme = "README.md"
@@ -2,7 +2,7 @@
2
2
  authors = ["Anthony MOI <m.anthony.moi@gmail.com>", "Nicolas Patry <patry.nicolas@protonmail.com>"]
3
3
  edition = "2018"
4
4
  name = "tokenizers"
5
- version = "0.22.2"
5
+ version = "0.22.2-rc0"
6
6
  homepage = "https://github.com/huggingface/tokenizers"
7
7
  repository = "https://github.com/huggingface/tokenizers"
8
8
  documentation = "https://docs.rs/tokenizers/"