tokenizers 0.21.2rc0__tar.gz → 0.21.4.dev0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of tokenizers might be problematic. Click here for more details.
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/PKG-INFO +1 -1
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/bindings/python/Cargo.lock +138 -65
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/bindings/python/Cargo.toml +1 -1
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/bindings/python/src/decoders.rs +2 -4
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/bindings/python/src/encoding.rs +9 -13
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/bindings/python/src/error.rs +3 -3
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/bindings/python/src/lib.rs +1 -2
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/bindings/python/src/models.rs +17 -25
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/bindings/python/src/normalizers.rs +5 -8
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/bindings/python/src/pre_tokenizers.rs +3 -6
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/bindings/python/src/processors.rs +2 -4
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/bindings/python/src/tokenizer.rs +12 -18
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/bindings/python/src/trainers.rs +7 -9
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/bindings/python/src/utils/normalization.rs +1 -2
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/Cargo.toml +1 -1
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/src/models/bpe/trainer.rs +5 -2
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/src/models/wordpiece/trainer.rs +7 -3
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/bindings/python/.cargo/config.toml +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/bindings/python/.gitignore +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/bindings/python/CHANGELOG.md +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/bindings/python/MANIFEST.in +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/bindings/python/Makefile +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/bindings/python/README.md +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/bindings/python/benches/test_tiktoken.py +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/bindings/python/conftest.py +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/bindings/python/examples/custom_components.py +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/bindings/python/examples/example.py +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/bindings/python/examples/train_bert_wordpiece.py +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/bindings/python/examples/train_bytelevel_bpe.py +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/bindings/python/examples/train_with_datasets.py +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/bindings/python/examples/using_the_visualizer.ipynb +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/bindings/python/py_src/tokenizers/__init__.py +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/bindings/python/py_src/tokenizers/__init__.pyi +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/bindings/python/py_src/tokenizers/decoders/__init__.py +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/bindings/python/py_src/tokenizers/decoders/__init__.pyi +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/bindings/python/py_src/tokenizers/implementations/__init__.py +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/bindings/python/py_src/tokenizers/implementations/base_tokenizer.py +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/bindings/python/py_src/tokenizers/implementations/bert_wordpiece.py +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/bindings/python/py_src/tokenizers/implementations/byte_level_bpe.py +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/bindings/python/py_src/tokenizers/implementations/char_level_bpe.py +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/bindings/python/py_src/tokenizers/implementations/sentencepiece_bpe.py +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/bindings/python/py_src/tokenizers/implementations/sentencepiece_unigram.py +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/bindings/python/py_src/tokenizers/models/__init__.py +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/bindings/python/py_src/tokenizers/models/__init__.pyi +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/bindings/python/py_src/tokenizers/normalizers/__init__.py +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/bindings/python/py_src/tokenizers/normalizers/__init__.pyi +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/bindings/python/py_src/tokenizers/pre_tokenizers/__init__.py +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/bindings/python/py_src/tokenizers/pre_tokenizers/__init__.pyi +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/bindings/python/py_src/tokenizers/processors/__init__.py +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/bindings/python/py_src/tokenizers/processors/__init__.pyi +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/bindings/python/py_src/tokenizers/tools/__init__.py +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/bindings/python/py_src/tokenizers/tools/visualizer-styles.css +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/bindings/python/py_src/tokenizers/tools/visualizer.py +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/bindings/python/py_src/tokenizers/trainers/__init__.py +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/bindings/python/py_src/tokenizers/trainers/__init__.pyi +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/bindings/python/rust-toolchain +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/bindings/python/scripts/convert.py +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/bindings/python/scripts/sentencepiece_extractor.py +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/bindings/python/scripts/spm_parity_check.py +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/bindings/python/setup.cfg +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/bindings/python/src/token.rs +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/bindings/python/src/utils/iterators.rs +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/bindings/python/src/utils/mod.rs +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/bindings/python/src/utils/pretokenization.rs +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/bindings/python/src/utils/regex.rs +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/bindings/python/src/utils/serde_pyo3.rs +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/bindings/python/stub.py +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/bindings/python/test.txt +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/bindings/python/tests/__init__.py +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/bindings/python/tests/bindings/__init__.py +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/bindings/python/tests/bindings/test_decoders.py +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/bindings/python/tests/bindings/test_encoding.py +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/bindings/python/tests/bindings/test_models.py +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/bindings/python/tests/bindings/test_normalizers.py +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/bindings/python/tests/bindings/test_pre_tokenizers.py +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/bindings/python/tests/bindings/test_processors.py +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/bindings/python/tests/bindings/test_tokenizer.py +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/bindings/python/tests/bindings/test_trainers.py +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/bindings/python/tests/documentation/__init__.py +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/bindings/python/tests/documentation/test_pipeline.py +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/bindings/python/tests/documentation/test_quicktour.py +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/bindings/python/tests/documentation/test_tutorial_train_from_iterators.py +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/bindings/python/tests/implementations/__init__.py +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/bindings/python/tests/implementations/test_base_tokenizer.py +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/bindings/python/tests/implementations/test_bert_wordpiece.py +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/bindings/python/tests/implementations/test_byte_level_bpe.py +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/bindings/python/tests/implementations/test_char_bpe.py +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/bindings/python/tests/implementations/test_sentencepiece.py +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/bindings/python/tests/test_serialization.py +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/bindings/python/tests/utils.py +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/py_src/tokenizers/__init__.py +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/py_src/tokenizers/__init__.pyi +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/py_src/tokenizers/decoders/__init__.py +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/py_src/tokenizers/decoders/__init__.pyi +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/py_src/tokenizers/implementations/__init__.py +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/py_src/tokenizers/implementations/base_tokenizer.py +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/py_src/tokenizers/implementations/bert_wordpiece.py +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/py_src/tokenizers/implementations/byte_level_bpe.py +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/py_src/tokenizers/implementations/char_level_bpe.py +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/py_src/tokenizers/implementations/sentencepiece_bpe.py +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/py_src/tokenizers/implementations/sentencepiece_unigram.py +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/py_src/tokenizers/models/__init__.py +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/py_src/tokenizers/models/__init__.pyi +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/py_src/tokenizers/normalizers/__init__.py +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/py_src/tokenizers/normalizers/__init__.pyi +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/py_src/tokenizers/pre_tokenizers/__init__.py +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/py_src/tokenizers/pre_tokenizers/__init__.pyi +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/py_src/tokenizers/processors/__init__.py +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/py_src/tokenizers/processors/__init__.pyi +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/py_src/tokenizers/tools/__init__.py +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/py_src/tokenizers/tools/visualizer-styles.css +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/py_src/tokenizers/tools/visualizer.py +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/py_src/tokenizers/trainers/__init__.py +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/py_src/tokenizers/trainers/__init__.pyi +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/pyproject.toml +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/CHANGELOG.md +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/LICENSE +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/Makefile +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/README.md +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/README.tpl +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/benches/bert_benchmark.rs +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/benches/bpe_benchmark.rs +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/benches/common/mod.rs +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/benches/layout_benchmark.rs +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/benches/llama3_benchmark.rs +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/benches/unigram_benchmark.rs +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/examples/encode_batch.rs +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/examples/serialization.rs +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/src/decoders/bpe.rs +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/src/decoders/byte_fallback.rs +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/src/decoders/ctc.rs +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/src/decoders/fuse.rs +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/src/decoders/mod.rs +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/src/decoders/sequence.rs +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/src/decoders/strip.rs +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/src/decoders/wordpiece.rs +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/src/lib.rs +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/src/models/bpe/mod.rs +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/src/models/bpe/model.rs +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/src/models/bpe/serialization.rs +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/src/models/bpe/word.rs +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/src/models/mod.rs +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/src/models/unigram/lattice.rs +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/src/models/unigram/mod.rs +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/src/models/unigram/model.rs +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/src/models/unigram/serialization.rs +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/src/models/unigram/trainer.rs +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/src/models/unigram/trie.rs +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/src/models/wordlevel/mod.rs +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/src/models/wordlevel/serialization.rs +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/src/models/wordlevel/trainer.rs +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/src/models/wordpiece/mod.rs +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/src/models/wordpiece/serialization.rs +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/src/normalizers/bert.rs +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/src/normalizers/byte_level.rs +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/src/normalizers/mod.rs +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/src/normalizers/precompiled.rs +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/src/normalizers/prepend.rs +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/src/normalizers/replace.rs +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/src/normalizers/strip.rs +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/src/normalizers/unicode.rs +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/src/normalizers/utils.rs +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/src/pre_tokenizers/bert.rs +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/src/pre_tokenizers/byte_level.rs +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/src/pre_tokenizers/delimiter.rs +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/src/pre_tokenizers/digits.rs +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/src/pre_tokenizers/fixed_length.rs +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/src/pre_tokenizers/metaspace.rs +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/src/pre_tokenizers/mod.rs +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/src/pre_tokenizers/punctuation.rs +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/src/pre_tokenizers/sequence.rs +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/src/pre_tokenizers/split.rs +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/src/pre_tokenizers/unicode_scripts/mod.rs +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/src/pre_tokenizers/unicode_scripts/pre_tokenizer.rs +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/src/pre_tokenizers/unicode_scripts/scripts.rs +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/src/pre_tokenizers/whitespace.rs +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/src/processors/bert.rs +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/src/processors/mod.rs +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/src/processors/roberta.rs +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/src/processors/sequence.rs +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/src/processors/template.rs +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/src/tokenizer/added_vocabulary.rs +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/src/tokenizer/encoding.rs +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/src/tokenizer/mod.rs +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/src/tokenizer/normalizer.rs +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/src/tokenizer/pattern.rs +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/src/tokenizer/pre_tokenizer.rs +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/src/tokenizer/serialization.rs +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/src/utils/cache.rs +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/src/utils/fancy.rs +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/src/utils/from_pretrained.rs +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/src/utils/iter.rs +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/src/utils/mod.rs +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/src/utils/onig.rs +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/src/utils/padding.rs +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/src/utils/parallelism.rs +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/src/utils/progress.rs +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/src/utils/truncation.rs +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/tests/added_tokens.rs +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/tests/common/mod.rs +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/tests/documentation.rs +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/tests/from_pretrained.rs +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/tests/offsets.rs +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/tests/serialization.rs +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/tests/stream.rs +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/tests/training.rs +0 -0
- {tokenizers-0.21.2rc0 → tokenizers-0.21.4.dev0}/tokenizers/tests/unigram.rs +0 -0
|
@@ -27,9 +27,9 @@ dependencies = [
|
|
|
27
27
|
|
|
28
28
|
[[package]]
|
|
29
29
|
name = "anstream"
|
|
30
|
-
version = "0.6.
|
|
30
|
+
version = "0.6.19"
|
|
31
31
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
32
|
-
checksum = "
|
|
32
|
+
checksum = "301af1932e46185686725e0fad2f8f2aa7da69dd70bf6ecc44d6b703844a3933"
|
|
33
33
|
dependencies = [
|
|
34
34
|
"anstyle",
|
|
35
35
|
"anstyle-parse",
|
|
@@ -42,44 +42,44 @@ dependencies = [
|
|
|
42
42
|
|
|
43
43
|
[[package]]
|
|
44
44
|
name = "anstyle"
|
|
45
|
-
version = "1.0.
|
|
45
|
+
version = "1.0.11"
|
|
46
46
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
47
|
-
checksum = "
|
|
47
|
+
checksum = "862ed96ca487e809f1c8e5a8447f6ee2cf102f846893800b20cebdf541fc6bbd"
|
|
48
48
|
|
|
49
49
|
[[package]]
|
|
50
50
|
name = "anstyle-parse"
|
|
51
|
-
version = "0.2.
|
|
51
|
+
version = "0.2.7"
|
|
52
52
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
53
|
-
checksum = "
|
|
53
|
+
checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2"
|
|
54
54
|
dependencies = [
|
|
55
55
|
"utf8parse",
|
|
56
56
|
]
|
|
57
57
|
|
|
58
58
|
[[package]]
|
|
59
59
|
name = "anstyle-query"
|
|
60
|
-
version = "1.1.
|
|
60
|
+
version = "1.1.3"
|
|
61
61
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
62
|
-
checksum = "
|
|
62
|
+
checksum = "6c8bdeb6047d8983be085bab0ba1472e6dc604e7041dbf6fcd5e71523014fae9"
|
|
63
63
|
dependencies = [
|
|
64
|
-
"windows-sys",
|
|
64
|
+
"windows-sys 0.59.0",
|
|
65
65
|
]
|
|
66
66
|
|
|
67
67
|
[[package]]
|
|
68
68
|
name = "anstyle-wincon"
|
|
69
|
-
version = "3.0.
|
|
69
|
+
version = "3.0.9"
|
|
70
70
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
71
|
-
checksum = "
|
|
71
|
+
checksum = "403f75924867bb1033c59fbf0797484329750cfbe3c4325cd33127941fabc882"
|
|
72
72
|
dependencies = [
|
|
73
73
|
"anstyle",
|
|
74
74
|
"once_cell_polyfill",
|
|
75
|
-
"windows-sys",
|
|
75
|
+
"windows-sys 0.59.0",
|
|
76
76
|
]
|
|
77
77
|
|
|
78
78
|
[[package]]
|
|
79
79
|
name = "autocfg"
|
|
80
|
-
version = "1.
|
|
80
|
+
version = "1.5.0"
|
|
81
81
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
82
|
-
checksum = "
|
|
82
|
+
checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8"
|
|
83
83
|
|
|
84
84
|
[[package]]
|
|
85
85
|
name = "base64"
|
|
@@ -95,9 +95,9 @@ checksum = "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967"
|
|
|
95
95
|
|
|
96
96
|
[[package]]
|
|
97
97
|
name = "bumpalo"
|
|
98
|
-
version = "3.
|
|
98
|
+
version = "3.19.0"
|
|
99
99
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
100
|
-
checksum = "
|
|
100
|
+
checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43"
|
|
101
101
|
|
|
102
102
|
[[package]]
|
|
103
103
|
name = "castaway"
|
|
@@ -110,24 +110,24 @@ dependencies = [
|
|
|
110
110
|
|
|
111
111
|
[[package]]
|
|
112
112
|
name = "cc"
|
|
113
|
-
version = "1.2.
|
|
113
|
+
version = "1.2.27"
|
|
114
114
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
115
|
-
checksum = "
|
|
115
|
+
checksum = "d487aa071b5f64da6f19a3e848e3578944b726ee5a4854b82172f02aa876bfdc"
|
|
116
116
|
dependencies = [
|
|
117
117
|
"shlex",
|
|
118
118
|
]
|
|
119
119
|
|
|
120
120
|
[[package]]
|
|
121
121
|
name = "cfg-if"
|
|
122
|
-
version = "1.0.
|
|
122
|
+
version = "1.0.1"
|
|
123
123
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
124
|
-
checksum = "
|
|
124
|
+
checksum = "9555578bc9e57714c812a1f84e4fc5b4d21fcb063490c624de019f7464c91268"
|
|
125
125
|
|
|
126
126
|
[[package]]
|
|
127
127
|
name = "colorchoice"
|
|
128
|
-
version = "1.0.
|
|
128
|
+
version = "1.0.4"
|
|
129
129
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
130
|
-
checksum = "
|
|
130
|
+
checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75"
|
|
131
131
|
|
|
132
132
|
[[package]]
|
|
133
133
|
name = "compact_str"
|
|
@@ -154,7 +154,7 @@ dependencies = [
|
|
|
154
154
|
"libc",
|
|
155
155
|
"once_cell",
|
|
156
156
|
"unicode-width",
|
|
157
|
-
"windows-sys",
|
|
157
|
+
"windows-sys 0.59.0",
|
|
158
158
|
]
|
|
159
159
|
|
|
160
160
|
[[package]]
|
|
@@ -294,12 +294,12 @@ dependencies = [
|
|
|
294
294
|
|
|
295
295
|
[[package]]
|
|
296
296
|
name = "errno"
|
|
297
|
-
version = "0.3.
|
|
297
|
+
version = "0.3.13"
|
|
298
298
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
299
|
-
checksum = "
|
|
299
|
+
checksum = "778e2ac28f6c47af28e4907f13ffd1e1ddbd400980a9abd7c8df189bf578a5ad"
|
|
300
300
|
dependencies = [
|
|
301
301
|
"libc",
|
|
302
|
-
"windows-sys",
|
|
302
|
+
"windows-sys 0.60.2",
|
|
303
303
|
]
|
|
304
304
|
|
|
305
305
|
[[package]]
|
|
@@ -389,9 +389,9 @@ checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c"
|
|
|
389
389
|
|
|
390
390
|
[[package]]
|
|
391
391
|
name = "jiff"
|
|
392
|
-
version = "0.2.
|
|
392
|
+
version = "0.2.15"
|
|
393
393
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
394
|
-
checksum = "
|
|
394
|
+
checksum = "be1f93b8b1eb69c77f24bbb0afdf66f54b632ee39af40ca21c4365a1d7347e49"
|
|
395
395
|
dependencies = [
|
|
396
396
|
"jiff-static",
|
|
397
397
|
"log",
|
|
@@ -402,9 +402,9 @@ dependencies = [
|
|
|
402
402
|
|
|
403
403
|
[[package]]
|
|
404
404
|
name = "jiff-static"
|
|
405
|
-
version = "0.2.
|
|
405
|
+
version = "0.2.15"
|
|
406
406
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
407
|
-
checksum = "
|
|
407
|
+
checksum = "03343451ff899767262ec32146f6d559dd759fdadf42ff0e227c7c48f72594b4"
|
|
408
408
|
dependencies = [
|
|
409
409
|
"proc-macro2",
|
|
410
410
|
"quote",
|
|
@@ -423,9 +423,9 @@ dependencies = [
|
|
|
423
423
|
|
|
424
424
|
[[package]]
|
|
425
425
|
name = "libc"
|
|
426
|
-
version = "0.2.
|
|
426
|
+
version = "0.2.174"
|
|
427
427
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
428
|
-
checksum = "
|
|
428
|
+
checksum = "1171693293099992e19cddea4e8b849964e9846f4acee11b3948bcc337be8776"
|
|
429
429
|
|
|
430
430
|
[[package]]
|
|
431
431
|
name = "linux-raw-sys"
|
|
@@ -441,9 +441,9 @@ checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94"
|
|
|
441
441
|
|
|
442
442
|
[[package]]
|
|
443
443
|
name = "macro_rules_attribute"
|
|
444
|
-
version = "0.2.
|
|
444
|
+
version = "0.2.2"
|
|
445
445
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
446
|
-
checksum = "
|
|
446
|
+
checksum = "65049d7923698040cd0b1ddcced9b0eb14dd22c5f86ae59c3740eab64a676520"
|
|
447
447
|
dependencies = [
|
|
448
448
|
"macro_rules_attribute-proc_macro",
|
|
449
449
|
"paste",
|
|
@@ -451,9 +451,9 @@ dependencies = [
|
|
|
451
451
|
|
|
452
452
|
[[package]]
|
|
453
453
|
name = "macro_rules_attribute-proc_macro"
|
|
454
|
-
version = "0.2.
|
|
454
|
+
version = "0.2.2"
|
|
455
455
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
456
|
-
checksum = "
|
|
456
|
+
checksum = "670fdfda89751bc4a84ac13eaa63e205cf0fd22b4c9a5fbfa085b63c1f1d3a30"
|
|
457
457
|
|
|
458
458
|
[[package]]
|
|
459
459
|
name = "matrixmultiply"
|
|
@@ -467,9 +467,9 @@ dependencies = [
|
|
|
467
467
|
|
|
468
468
|
[[package]]
|
|
469
469
|
name = "memchr"
|
|
470
|
-
version = "2.7.
|
|
470
|
+
version = "2.7.5"
|
|
471
471
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
472
|
-
checksum = "
|
|
472
|
+
checksum = "32a282da65faaf38286cf3be983213fcf1d2e2a58700e808f83f4ea9a4804bc0"
|
|
473
473
|
|
|
474
474
|
[[package]]
|
|
475
475
|
name = "memoffset"
|
|
@@ -629,9 +629,9 @@ checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c"
|
|
|
629
629
|
|
|
630
630
|
[[package]]
|
|
631
631
|
name = "portable-atomic"
|
|
632
|
-
version = "1.11.
|
|
632
|
+
version = "1.11.1"
|
|
633
633
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
634
|
-
checksum = "
|
|
634
|
+
checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483"
|
|
635
635
|
|
|
636
636
|
[[package]]
|
|
637
637
|
name = "portable-atomic-util"
|
|
@@ -733,9 +733,9 @@ dependencies = [
|
|
|
733
733
|
|
|
734
734
|
[[package]]
|
|
735
735
|
name = "r-efi"
|
|
736
|
-
version = "5.
|
|
736
|
+
version = "5.3.0"
|
|
737
737
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
738
|
-
checksum = "
|
|
738
|
+
checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f"
|
|
739
739
|
|
|
740
740
|
[[package]]
|
|
741
741
|
name = "rand"
|
|
@@ -848,7 +848,7 @@ dependencies = [
|
|
|
848
848
|
"errno",
|
|
849
849
|
"libc",
|
|
850
850
|
"linux-raw-sys",
|
|
851
|
-
"windows-sys",
|
|
851
|
+
"windows-sys 0.59.0",
|
|
852
852
|
]
|
|
853
853
|
|
|
854
854
|
[[package]]
|
|
@@ -903,9 +903,9 @@ checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
|
|
|
903
903
|
|
|
904
904
|
[[package]]
|
|
905
905
|
name = "smallvec"
|
|
906
|
-
version = "1.15.
|
|
906
|
+
version = "1.15.1"
|
|
907
907
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
908
|
-
checksum = "
|
|
908
|
+
checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03"
|
|
909
909
|
|
|
910
910
|
[[package]]
|
|
911
911
|
name = "spm_precompiled"
|
|
@@ -933,9 +933,9 @@ checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
|
|
|
933
933
|
|
|
934
934
|
[[package]]
|
|
935
935
|
name = "syn"
|
|
936
|
-
version = "2.0.
|
|
936
|
+
version = "2.0.104"
|
|
937
937
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
938
|
-
checksum = "
|
|
938
|
+
checksum = "17b6f705963418cdb9927482fa304bc562ece2fdd4f616084c50b7023b435a40"
|
|
939
939
|
dependencies = [
|
|
940
940
|
"proc-macro2",
|
|
941
941
|
"quote",
|
|
@@ -958,7 +958,7 @@ dependencies = [
|
|
|
958
958
|
"getrandom",
|
|
959
959
|
"once_cell",
|
|
960
960
|
"rustix",
|
|
961
|
-
"windows-sys",
|
|
961
|
+
"windows-sys 0.59.0",
|
|
962
962
|
]
|
|
963
963
|
|
|
964
964
|
[[package]]
|
|
@@ -983,7 +983,7 @@ dependencies = [
|
|
|
983
983
|
|
|
984
984
|
[[package]]
|
|
985
985
|
name = "tokenizers"
|
|
986
|
-
version = "0.21.
|
|
986
|
+
version = "0.21.4-dev.0"
|
|
987
987
|
dependencies = [
|
|
988
988
|
"ahash",
|
|
989
989
|
"aho-corasick",
|
|
@@ -1015,7 +1015,7 @@ dependencies = [
|
|
|
1015
1015
|
|
|
1016
1016
|
[[package]]
|
|
1017
1017
|
name = "tokenizers-python"
|
|
1018
|
-
version = "0.21.
|
|
1018
|
+
version = "0.21.4-dev.0"
|
|
1019
1019
|
dependencies = [
|
|
1020
1020
|
"ahash",
|
|
1021
1021
|
"env_logger",
|
|
@@ -1054,9 +1054,9 @@ checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493"
|
|
|
1054
1054
|
|
|
1055
1055
|
[[package]]
|
|
1056
1056
|
name = "unicode-width"
|
|
1057
|
-
version = "0.2.
|
|
1057
|
+
version = "0.2.1"
|
|
1058
1058
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
1059
|
-
checksum = "
|
|
1059
|
+
checksum = "4a1a07cc7db3810833284e8d372ccdc6da29741639ecc70c9ec107df0fa6154c"
|
|
1060
1060
|
|
|
1061
1061
|
[[package]]
|
|
1062
1062
|
name = "unicode_categories"
|
|
@@ -1164,7 +1164,16 @@ version = "0.59.0"
|
|
|
1164
1164
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
1165
1165
|
checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b"
|
|
1166
1166
|
dependencies = [
|
|
1167
|
-
"windows-targets",
|
|
1167
|
+
"windows-targets 0.52.6",
|
|
1168
|
+
]
|
|
1169
|
+
|
|
1170
|
+
[[package]]
|
|
1171
|
+
name = "windows-sys"
|
|
1172
|
+
version = "0.60.2"
|
|
1173
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
1174
|
+
checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb"
|
|
1175
|
+
dependencies = [
|
|
1176
|
+
"windows-targets 0.53.2",
|
|
1168
1177
|
]
|
|
1169
1178
|
|
|
1170
1179
|
[[package]]
|
|
@@ -1173,14 +1182,30 @@ version = "0.52.6"
|
|
|
1173
1182
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
1174
1183
|
checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"
|
|
1175
1184
|
dependencies = [
|
|
1176
|
-
"windows_aarch64_gnullvm",
|
|
1177
|
-
"windows_aarch64_msvc",
|
|
1178
|
-
"windows_i686_gnu",
|
|
1179
|
-
"windows_i686_gnullvm",
|
|
1180
|
-
"windows_i686_msvc",
|
|
1181
|
-
"windows_x86_64_gnu",
|
|
1182
|
-
"windows_x86_64_gnullvm",
|
|
1183
|
-
"windows_x86_64_msvc",
|
|
1185
|
+
"windows_aarch64_gnullvm 0.52.6",
|
|
1186
|
+
"windows_aarch64_msvc 0.52.6",
|
|
1187
|
+
"windows_i686_gnu 0.52.6",
|
|
1188
|
+
"windows_i686_gnullvm 0.52.6",
|
|
1189
|
+
"windows_i686_msvc 0.52.6",
|
|
1190
|
+
"windows_x86_64_gnu 0.52.6",
|
|
1191
|
+
"windows_x86_64_gnullvm 0.52.6",
|
|
1192
|
+
"windows_x86_64_msvc 0.52.6",
|
|
1193
|
+
]
|
|
1194
|
+
|
|
1195
|
+
[[package]]
|
|
1196
|
+
name = "windows-targets"
|
|
1197
|
+
version = "0.53.2"
|
|
1198
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
1199
|
+
checksum = "c66f69fcc9ce11da9966ddb31a40968cad001c5bedeb5c2b82ede4253ab48aef"
|
|
1200
|
+
dependencies = [
|
|
1201
|
+
"windows_aarch64_gnullvm 0.53.0",
|
|
1202
|
+
"windows_aarch64_msvc 0.53.0",
|
|
1203
|
+
"windows_i686_gnu 0.53.0",
|
|
1204
|
+
"windows_i686_gnullvm 0.53.0",
|
|
1205
|
+
"windows_i686_msvc 0.53.0",
|
|
1206
|
+
"windows_x86_64_gnu 0.53.0",
|
|
1207
|
+
"windows_x86_64_gnullvm 0.53.0",
|
|
1208
|
+
"windows_x86_64_msvc 0.53.0",
|
|
1184
1209
|
]
|
|
1185
1210
|
|
|
1186
1211
|
[[package]]
|
|
@@ -1189,48 +1214,96 @@ version = "0.52.6"
|
|
|
1189
1214
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
1190
1215
|
checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
|
|
1191
1216
|
|
|
1217
|
+
[[package]]
|
|
1218
|
+
name = "windows_aarch64_gnullvm"
|
|
1219
|
+
version = "0.53.0"
|
|
1220
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
1221
|
+
checksum = "86b8d5f90ddd19cb4a147a5fa63ca848db3df085e25fee3cc10b39b6eebae764"
|
|
1222
|
+
|
|
1192
1223
|
[[package]]
|
|
1193
1224
|
name = "windows_aarch64_msvc"
|
|
1194
1225
|
version = "0.52.6"
|
|
1195
1226
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
1196
1227
|
checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
|
|
1197
1228
|
|
|
1229
|
+
[[package]]
|
|
1230
|
+
name = "windows_aarch64_msvc"
|
|
1231
|
+
version = "0.53.0"
|
|
1232
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
1233
|
+
checksum = "c7651a1f62a11b8cbd5e0d42526e55f2c99886c77e007179efff86c2b137e66c"
|
|
1234
|
+
|
|
1198
1235
|
[[package]]
|
|
1199
1236
|
name = "windows_i686_gnu"
|
|
1200
1237
|
version = "0.52.6"
|
|
1201
1238
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
1202
1239
|
checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"
|
|
1203
1240
|
|
|
1241
|
+
[[package]]
|
|
1242
|
+
name = "windows_i686_gnu"
|
|
1243
|
+
version = "0.53.0"
|
|
1244
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
1245
|
+
checksum = "c1dc67659d35f387f5f6c479dc4e28f1d4bb90ddd1a5d3da2e5d97b42d6272c3"
|
|
1246
|
+
|
|
1204
1247
|
[[package]]
|
|
1205
1248
|
name = "windows_i686_gnullvm"
|
|
1206
1249
|
version = "0.52.6"
|
|
1207
1250
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
1208
1251
|
checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
|
|
1209
1252
|
|
|
1253
|
+
[[package]]
|
|
1254
|
+
name = "windows_i686_gnullvm"
|
|
1255
|
+
version = "0.53.0"
|
|
1256
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
1257
|
+
checksum = "9ce6ccbdedbf6d6354471319e781c0dfef054c81fbc7cf83f338a4296c0cae11"
|
|
1258
|
+
|
|
1210
1259
|
[[package]]
|
|
1211
1260
|
name = "windows_i686_msvc"
|
|
1212
1261
|
version = "0.52.6"
|
|
1213
1262
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
1214
1263
|
checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
|
|
1215
1264
|
|
|
1265
|
+
[[package]]
|
|
1266
|
+
name = "windows_i686_msvc"
|
|
1267
|
+
version = "0.53.0"
|
|
1268
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
1269
|
+
checksum = "581fee95406bb13382d2f65cd4a908ca7b1e4c2f1917f143ba16efe98a589b5d"
|
|
1270
|
+
|
|
1216
1271
|
[[package]]
|
|
1217
1272
|
name = "windows_x86_64_gnu"
|
|
1218
1273
|
version = "0.52.6"
|
|
1219
1274
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
1220
1275
|
checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
|
|
1221
1276
|
|
|
1277
|
+
[[package]]
|
|
1278
|
+
name = "windows_x86_64_gnu"
|
|
1279
|
+
version = "0.53.0"
|
|
1280
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
1281
|
+
checksum = "2e55b5ac9ea33f2fc1716d1742db15574fd6fc8dadc51caab1c16a3d3b4190ba"
|
|
1282
|
+
|
|
1222
1283
|
[[package]]
|
|
1223
1284
|
name = "windows_x86_64_gnullvm"
|
|
1224
1285
|
version = "0.52.6"
|
|
1225
1286
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
1226
1287
|
checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
|
|
1227
1288
|
|
|
1289
|
+
[[package]]
|
|
1290
|
+
name = "windows_x86_64_gnullvm"
|
|
1291
|
+
version = "0.53.0"
|
|
1292
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
1293
|
+
checksum = "0a6e035dd0599267ce1ee132e51c27dd29437f63325753051e71dd9e42406c57"
|
|
1294
|
+
|
|
1228
1295
|
[[package]]
|
|
1229
1296
|
name = "windows_x86_64_msvc"
|
|
1230
1297
|
version = "0.52.6"
|
|
1231
1298
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
1232
1299
|
checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
|
|
1233
1300
|
|
|
1301
|
+
[[package]]
|
|
1302
|
+
name = "windows_x86_64_msvc"
|
|
1303
|
+
version = "0.53.0"
|
|
1304
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
1305
|
+
checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486"
|
|
1306
|
+
|
|
1234
1307
|
[[package]]
|
|
1235
1308
|
name = "wit-bindgen-rt"
|
|
1236
1309
|
version = "0.39.0"
|
|
@@ -1242,18 +1315,18 @@ dependencies = [
|
|
|
1242
1315
|
|
|
1243
1316
|
[[package]]
|
|
1244
1317
|
name = "zerocopy"
|
|
1245
|
-
version = "0.8.
|
|
1318
|
+
version = "0.8.26"
|
|
1246
1319
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
1247
|
-
checksum = "
|
|
1320
|
+
checksum = "1039dd0d3c310cf05de012d8a39ff557cb0d23087fd44cad61df08fc31907a2f"
|
|
1248
1321
|
dependencies = [
|
|
1249
1322
|
"zerocopy-derive",
|
|
1250
1323
|
]
|
|
1251
1324
|
|
|
1252
1325
|
[[package]]
|
|
1253
1326
|
name = "zerocopy-derive"
|
|
1254
|
-
version = "0.8.
|
|
1327
|
+
version = "0.8.26"
|
|
1255
1328
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
1256
|
-
checksum = "
|
|
1329
|
+
checksum = "9ecf5b4cc5364572d7f4c329661bcc82724222973f2cab6f050a4e5c22f75181"
|
|
1257
1330
|
dependencies = [
|
|
1258
1331
|
"proc-macro2",
|
|
1259
1332
|
"quote",
|
|
@@ -107,8 +107,7 @@ impl PyDecoder {
|
|
|
107
107
|
fn __getstate__(&self, py: Python) -> PyResult<PyObject> {
|
|
108
108
|
let data = serde_json::to_string(&self.decoder).map_err(|e| {
|
|
109
109
|
exceptions::PyException::new_err(format!(
|
|
110
|
-
"Error while attempting to pickle Decoder: {}"
|
|
111
|
-
e
|
|
110
|
+
"Error while attempting to pickle Decoder: {e}"
|
|
112
111
|
))
|
|
113
112
|
})?;
|
|
114
113
|
Ok(PyBytes::new(py, data.as_bytes()).into())
|
|
@@ -119,8 +118,7 @@ impl PyDecoder {
|
|
|
119
118
|
Ok(s) => {
|
|
120
119
|
self.decoder = serde_json::from_slice(s).map_err(|e| {
|
|
121
120
|
exceptions::PyException::new_err(format!(
|
|
122
|
-
"Error while attempting to unpickle Decoder: {}"
|
|
123
|
-
e
|
|
121
|
+
"Error while attempting to unpickle Decoder: {e}"
|
|
124
122
|
))
|
|
125
123
|
})?;
|
|
126
124
|
Ok(())
|
|
@@ -33,8 +33,7 @@ impl PyEncoding {
|
|
|
33
33
|
fn __getstate__(&self, py: Python) -> PyResult<PyObject> {
|
|
34
34
|
let data = serde_json::to_string(&self.encoding).map_err(|e| {
|
|
35
35
|
exceptions::PyException::new_err(format!(
|
|
36
|
-
"Error while attempting to pickle Encoding: {}"
|
|
37
|
-
e
|
|
36
|
+
"Error while attempting to pickle Encoding: {e}"
|
|
38
37
|
))
|
|
39
38
|
})?;
|
|
40
39
|
Ok(PyBytes::new(py, data.as_bytes()).into())
|
|
@@ -45,8 +44,7 @@ impl PyEncoding {
|
|
|
45
44
|
Ok(s) => {
|
|
46
45
|
self.encoding = serde_json::from_slice(s).map_err(|e| {
|
|
47
46
|
exceptions::PyException::new_err(format!(
|
|
48
|
-
"Error while attempting to unpickle Encoding: {}"
|
|
49
|
-
e
|
|
47
|
+
"Error while attempting to unpickle Encoding: {e}"
|
|
50
48
|
))
|
|
51
49
|
})?;
|
|
52
50
|
Ok(())
|
|
@@ -407,9 +405,8 @@ impl PyEncoding {
|
|
|
407
405
|
"left" => Ok(PaddingDirection::Left),
|
|
408
406
|
"right" => Ok(PaddingDirection::Right),
|
|
409
407
|
other => Err(PyError(format!(
|
|
410
|
-
"Unknown `direction`: `{}`. Use \
|
|
411
|
-
one of `left` or `right`"
|
|
412
|
-
other
|
|
408
|
+
"Unknown `direction`: `{other}`. Use \
|
|
409
|
+
one of `left` or `right`"
|
|
413
410
|
))
|
|
414
411
|
.into_pyerr::<exceptions::PyValueError>()),
|
|
415
412
|
}?;
|
|
@@ -417,7 +414,7 @@ impl PyEncoding {
|
|
|
417
414
|
"pad_id" => pad_id = value.extract()?,
|
|
418
415
|
"pad_type_id" => pad_type_id = value.extract()?,
|
|
419
416
|
"pad_token" => pad_token = value.extract()?,
|
|
420
|
-
_ => println!("Ignored unknown kwarg option {}"
|
|
417
|
+
_ => println!("Ignored unknown kwarg option {key}"),
|
|
421
418
|
}
|
|
422
419
|
}
|
|
423
420
|
}
|
|
@@ -446,11 +443,10 @@ impl PyEncoding {
|
|
|
446
443
|
let tdir = match direction {
|
|
447
444
|
"left" => Ok(TruncationDirection::Left),
|
|
448
445
|
"right" => Ok(TruncationDirection::Right),
|
|
449
|
-
_ => Err(
|
|
450
|
-
"Invalid truncation direction value : {}"
|
|
451
|
-
|
|
452
|
-
)
|
|
453
|
-
.into_pyerr::<exceptions::PyValueError>()),
|
|
446
|
+
_ => Err(
|
|
447
|
+
PyError(format!("Invalid truncation direction value : {direction}"))
|
|
448
|
+
.into_pyerr::<exceptions::PyValueError>(),
|
|
449
|
+
),
|
|
454
450
|
}?;
|
|
455
451
|
|
|
456
452
|
self.encoding.truncate(max_length, stride, tdir);
|
|
@@ -13,7 +13,7 @@ impl PyError {
|
|
|
13
13
|
PyError(String::from(s))
|
|
14
14
|
}
|
|
15
15
|
pub fn into_pyerr<T: PyTypeInfo>(self) -> PyErr {
|
|
16
|
-
PyErr::new::<T, _>(format!("{}"
|
|
16
|
+
PyErr::new::<T, _>(format!("{self}"))
|
|
17
17
|
}
|
|
18
18
|
}
|
|
19
19
|
impl Display for PyError {
|
|
@@ -26,7 +26,7 @@ impl std::error::Error for PyError {}
|
|
|
26
26
|
pub struct ToPyResult<T>(pub Result<T>);
|
|
27
27
|
impl<T> From<ToPyResult<T>> for PyResult<T> {
|
|
28
28
|
fn from(v: ToPyResult<T>) -> Self {
|
|
29
|
-
v.0.map_err(|e| exceptions::PyException::new_err(format!("{}"
|
|
29
|
+
v.0.map_err(|e| exceptions::PyException::new_err(format!("{e}")))
|
|
30
30
|
}
|
|
31
31
|
}
|
|
32
32
|
impl<T> ToPyResult<T> {
|
|
@@ -37,6 +37,6 @@ impl<T> ToPyResult<T> {
|
|
|
37
37
|
|
|
38
38
|
pub(crate) fn deprecation_warning(py: Python<'_>, version: &str, message: &str) -> PyResult<()> {
|
|
39
39
|
let deprecation_warning = py.import("builtins")?.getattr("DeprecationWarning")?;
|
|
40
|
-
let full_message = format!("Deprecated in {}: {}"
|
|
40
|
+
let full_message = format!("Deprecated in {version}: {message}");
|
|
41
41
|
pyo3::PyErr::warn(py, &deprecation_warning, &CString::new(full_message)?, 0)
|
|
42
42
|
}
|
|
@@ -38,8 +38,7 @@ extern "C" fn child_after_fork() {
|
|
|
38
38
|
eprintln!("To disable this warning, you can either:");
|
|
39
39
|
eprintln!(
|
|
40
40
|
"\t- Avoid using `tokenizers` before the fork if possible\n\
|
|
41
|
-
\t- Explicitly set the environment variable {}=(true | false)"
|
|
42
|
-
ENV_VARIABLE
|
|
41
|
+
\t- Explicitly set the environment variable {ENV_VARIABLE}=(true | false)"
|
|
43
42
|
);
|
|
44
43
|
set_parallelism(false);
|
|
45
44
|
}
|