tokenizers 0.22.2__tar.gz → 0.22.2rc0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/PKG-INFO +1 -1
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/Cargo.lock +2 -2
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/Cargo.toml +1 -1
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/Cargo.toml +1 -1
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/.cargo/config.toml +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/.gitignore +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/CHANGELOG.md +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/MANIFEST.in +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/Makefile +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/README.md +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/benches/test_tiktoken.py +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/conftest.py +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/docs/pyo3.md +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/examples/custom_components.py +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/examples/example.py +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/examples/train_bert_wordpiece.py +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/examples/train_bytelevel_bpe.py +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/examples/train_with_datasets.py +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/examples/using_the_visualizer.ipynb +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/py_src/tokenizers/__init__.py +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/py_src/tokenizers/__init__.pyi +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/py_src/tokenizers/decoders/__init__.py +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/py_src/tokenizers/decoders/__init__.pyi +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/py_src/tokenizers/implementations/__init__.py +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/py_src/tokenizers/implementations/base_tokenizer.py +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/py_src/tokenizers/implementations/bert_wordpiece.py +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/py_src/tokenizers/implementations/byte_level_bpe.py +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/py_src/tokenizers/implementations/char_level_bpe.py +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/py_src/tokenizers/implementations/sentencepiece_bpe.py +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/py_src/tokenizers/implementations/sentencepiece_unigram.py +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/py_src/tokenizers/models/__init__.py +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/py_src/tokenizers/models/__init__.pyi +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/py_src/tokenizers/normalizers/__init__.py +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/py_src/tokenizers/normalizers/__init__.pyi +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/py_src/tokenizers/pre_tokenizers/__init__.py +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/py_src/tokenizers/pre_tokenizers/__init__.pyi +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/py_src/tokenizers/processors/__init__.py +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/py_src/tokenizers/processors/__init__.pyi +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/py_src/tokenizers/tokenizers.pyi +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/py_src/tokenizers/tools/__init__.py +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/py_src/tokenizers/tools/visualizer-styles.css +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/py_src/tokenizers/tools/visualizer.py +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/py_src/tokenizers/trainers/__init__.py +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/py_src/tokenizers/trainers/__init__.pyi +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/rust-toolchain +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/scripts/convert.py +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/scripts/sentencepiece_extractor.py +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/scripts/spm_parity_check.py +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/setup.cfg +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/src/decoders.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/src/encoding.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/src/error.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/src/lib.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/src/models.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/src/normalizers.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/src/pre_tokenizers.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/src/processors.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/src/token.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/src/tokenizer.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/src/trainers.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/src/utils/iterators.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/src/utils/mod.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/src/utils/normalization.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/src/utils/pretokenization.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/src/utils/regex.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/src/utils/serde_pyo3.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/stub.py +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/test.txt +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/tests/__init__.py +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/tests/bindings/__init__.py +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/tests/bindings/test_decoders.py +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/tests/bindings/test_encoding.py +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/tests/bindings/test_models.py +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/tests/bindings/test_normalizers.py +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/tests/bindings/test_pre_tokenizers.py +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/tests/bindings/test_processors.py +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/tests/bindings/test_tokenizer.py +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/tests/bindings/test_trainers.py +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/tests/documentation/__init__.py +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/tests/documentation/test_pipeline.py +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/tests/documentation/test_quicktour.py +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/tests/documentation/test_tutorial_train_from_iterators.py +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/tests/implementations/__init__.py +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/tests/implementations/test_base_tokenizer.py +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/tests/implementations/test_bert_wordpiece.py +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/tests/implementations/test_byte_level_bpe.py +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/tests/implementations/test_char_bpe.py +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/tests/implementations/test_sentencepiece.py +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/tests/test_serialization.py +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/tests/utils.py +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/py_src/tokenizers/__init__.py +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/py_src/tokenizers/__init__.pyi +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/py_src/tokenizers/decoders/__init__.py +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/py_src/tokenizers/decoders/__init__.pyi +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/py_src/tokenizers/implementations/__init__.py +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/py_src/tokenizers/implementations/base_tokenizer.py +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/py_src/tokenizers/implementations/bert_wordpiece.py +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/py_src/tokenizers/implementations/byte_level_bpe.py +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/py_src/tokenizers/implementations/char_level_bpe.py +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/py_src/tokenizers/implementations/sentencepiece_bpe.py +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/py_src/tokenizers/implementations/sentencepiece_unigram.py +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/py_src/tokenizers/models/__init__.py +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/py_src/tokenizers/models/__init__.pyi +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/py_src/tokenizers/normalizers/__init__.py +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/py_src/tokenizers/normalizers/__init__.pyi +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/py_src/tokenizers/pre_tokenizers/__init__.py +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/py_src/tokenizers/pre_tokenizers/__init__.pyi +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/py_src/tokenizers/processors/__init__.py +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/py_src/tokenizers/processors/__init__.pyi +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/py_src/tokenizers/tokenizers.pyi +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/py_src/tokenizers/tools/__init__.py +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/py_src/tokenizers/tools/visualizer-styles.css +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/py_src/tokenizers/tools/visualizer.py +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/py_src/tokenizers/trainers/__init__.py +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/py_src/tokenizers/trainers/__init__.pyi +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/pyproject.toml +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/CHANGELOG.md +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/LICENSE +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/Makefile +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/README.md +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/README.tpl +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/benches/added_vocab_deserialize.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/benches/bert_benchmark.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/benches/bpe_benchmark.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/benches/common/mod.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/benches/layout_benchmark.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/benches/llama3_benchmark.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/benches/unigram_benchmark.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/examples/encode_batch.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/examples/serialization.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/decoders/bpe.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/decoders/byte_fallback.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/decoders/ctc.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/decoders/fuse.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/decoders/mod.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/decoders/sequence.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/decoders/strip.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/decoders/wordpiece.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/lib.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/models/bpe/mod.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/models/bpe/model.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/models/bpe/serialization.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/models/bpe/trainer.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/models/bpe/word.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/models/mod.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/models/unigram/lattice.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/models/unigram/mod.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/models/unigram/model.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/models/unigram/serialization.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/models/unigram/trainer.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/models/unigram/trie.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/models/wordlevel/mod.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/models/wordlevel/serialization.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/models/wordlevel/trainer.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/models/wordpiece/mod.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/models/wordpiece/serialization.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/models/wordpiece/trainer.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/normalizers/bert.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/normalizers/byte_level.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/normalizers/mod.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/normalizers/precompiled.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/normalizers/prepend.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/normalizers/replace.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/normalizers/strip.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/normalizers/unicode.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/normalizers/utils.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/pre_tokenizers/bert.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/pre_tokenizers/byte_level.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/pre_tokenizers/delimiter.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/pre_tokenizers/digits.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/pre_tokenizers/fixed_length.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/pre_tokenizers/metaspace.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/pre_tokenizers/mod.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/pre_tokenizers/punctuation.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/pre_tokenizers/sequence.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/pre_tokenizers/split.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/pre_tokenizers/unicode_scripts/mod.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/pre_tokenizers/unicode_scripts/pre_tokenizer.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/pre_tokenizers/unicode_scripts/scripts.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/pre_tokenizers/whitespace.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/processors/bert.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/processors/mod.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/processors/roberta.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/processors/sequence.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/processors/template.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/tokenizer/added_vocabulary.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/tokenizer/encoding.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/tokenizer/mod.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/tokenizer/normalizer.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/tokenizer/pattern.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/tokenizer/pre_tokenizer.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/tokenizer/serialization.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/utils/cache.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/utils/fancy.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/utils/from_pretrained.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/utils/iter.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/utils/mod.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/utils/onig.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/utils/padding.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/utils/parallelism.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/utils/progress.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/utils/truncation.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/tests/added_tokens.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/tests/common/mod.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/tests/documentation.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/tests/from_pretrained.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/tests/offsets.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/tests/serialization.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/tests/stream.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/tests/training.rs +0 -0
- {tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/tests/unigram.rs +0 -0
|
@@ -1138,7 +1138,7 @@ dependencies = [
|
|
|
1138
1138
|
|
|
1139
1139
|
[[package]]
|
|
1140
1140
|
name = "tokenizers"
|
|
1141
|
-
version = "0.22.2"
|
|
1141
|
+
version = "0.22.2-rc0"
|
|
1142
1142
|
dependencies = [
|
|
1143
1143
|
"ahash",
|
|
1144
1144
|
"aho-corasick",
|
|
@@ -1170,7 +1170,7 @@ dependencies = [
|
|
|
1170
1170
|
|
|
1171
1171
|
[[package]]
|
|
1172
1172
|
name = "tokenizers-python"
|
|
1173
|
-
version = "0.22.2"
|
|
1173
|
+
version = "0.22.2-rc0"
|
|
1174
1174
|
dependencies = [
|
|
1175
1175
|
"ahash",
|
|
1176
1176
|
"env_logger",
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
authors = ["Anthony MOI <m.anthony.moi@gmail.com>", "Nicolas Patry <patry.nicolas@protonmail.com>"]
|
|
3
3
|
edition = "2018"
|
|
4
4
|
name = "tokenizers"
|
|
5
|
-
version = "0.22.2"
|
|
5
|
+
version = "0.22.2-rc0"
|
|
6
6
|
homepage = "https://github.com/huggingface/tokenizers"
|
|
7
7
|
repository = "https://github.com/huggingface/tokenizers"
|
|
8
8
|
documentation = "https://docs.rs/tokenizers/"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/examples/using_the_visualizer.ipynb
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/py_src/tokenizers/decoders/__init__.py
RENAMED
|
File without changes
|
{tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/py_src/tokenizers/decoders/__init__.pyi
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/py_src/tokenizers/models/__init__.py
RENAMED
|
File without changes
|
{tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/py_src/tokenizers/models/__init__.pyi
RENAMED
|
File without changes
|
{tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/py_src/tokenizers/normalizers/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/py_src/tokenizers/processors/__init__.py
RENAMED
|
File without changes
|
{tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/py_src/tokenizers/processors/__init__.pyi
RENAMED
|
File without changes
|
|
File without changes
|
{tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/py_src/tokenizers/tools/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
{tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/py_src/tokenizers/tools/visualizer.py
RENAMED
|
File without changes
|
{tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/py_src/tokenizers/trainers/__init__.py
RENAMED
|
File without changes
|
{tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/py_src/tokenizers/trainers/__init__.pyi
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/scripts/sentencepiece_extractor.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/tests/bindings/test_normalizers.py
RENAMED
|
File without changes
|
{tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/tests/bindings/test_pre_tokenizers.py
RENAMED
|
File without changes
|
{tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/tests/bindings/test_processors.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/tests/documentation/test_pipeline.py
RENAMED
|
File without changes
|
{tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/tests/documentation/test_quicktour.py
RENAMED
|
File without changes
|
|
File without changes
|
{tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/tests/implementations/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{tokenizers-0.22.2 → tokenizers-0.22.2rc0}/bindings/python/tests/implementations/test_char_bpe.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{tokenizers-0.22.2 → tokenizers-0.22.2rc0}/py_src/tokenizers/implementations/base_tokenizer.py
RENAMED
|
File without changes
|
{tokenizers-0.22.2 → tokenizers-0.22.2rc0}/py_src/tokenizers/implementations/bert_wordpiece.py
RENAMED
|
File without changes
|
{tokenizers-0.22.2 → tokenizers-0.22.2rc0}/py_src/tokenizers/implementations/byte_level_bpe.py
RENAMED
|
File without changes
|
{tokenizers-0.22.2 → tokenizers-0.22.2rc0}/py_src/tokenizers/implementations/char_level_bpe.py
RENAMED
|
File without changes
|
{tokenizers-0.22.2 → tokenizers-0.22.2rc0}/py_src/tokenizers/implementations/sentencepiece_bpe.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/pre_tokenizers/unicode_scripts/mod.rs
RENAMED
|
File without changes
|
|
File without changes
|
{tokenizers-0.22.2 → tokenizers-0.22.2rc0}/tokenizers/src/pre_tokenizers/unicode_scripts/scripts.rs
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|