tokenizers 0.21.4.dev0__tar.gz → 0.22.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/PKG-INFO +3 -3
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/bindings/python/Cargo.lock +396 -100
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/bindings/python/Cargo.toml +5 -1
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1/bindings/python}/py_src/tokenizers/__init__.pyi +124 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1/bindings/python}/py_src/tokenizers/decoders/__init__.pyi +1 -1
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/bindings/python/py_src/tokenizers/implementations/base_tokenizer.py +41 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/bindings/python/py_src/tokenizers/tools/visualizer.py +1 -1
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/bindings/python/py_src/tokenizers/trainers/__init__.pyi +17 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/bindings/python/src/decoders.rs +30 -7
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/bindings/python/src/lib.rs +13 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/bindings/python/src/processors.rs +4 -4
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/bindings/python/src/tokenizer.rs +365 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/bindings/python/src/trainers.rs +10 -4
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/bindings/python/tests/bindings/test_tokenizer.py +362 -5
- {tokenizers-0.21.4.dev0/bindings/python → tokenizers-0.22.1}/py_src/tokenizers/__init__.pyi +124 -0
- {tokenizers-0.21.4.dev0/bindings/python → tokenizers-0.22.1}/py_src/tokenizers/decoders/__init__.pyi +1 -1
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/py_src/tokenizers/implementations/base_tokenizer.py +41 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/py_src/tokenizers/tools/visualizer.py +1 -1
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/py_src/tokenizers/trainers/__init__.pyi +17 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/pyproject.toml +2 -2
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/Cargo.toml +1 -1
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/src/models/bpe/trainer.rs +1 -4
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/src/models/unigram/model.rs +1 -1
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/src/models/unigram/trie.rs +1 -1
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/src/models/wordpiece/mod.rs +17 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/src/tokenizer/mod.rs +23 -7
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/bindings/python/.cargo/config.toml +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/bindings/python/.gitignore +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/bindings/python/CHANGELOG.md +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/bindings/python/MANIFEST.in +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/bindings/python/Makefile +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/bindings/python/README.md +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/bindings/python/benches/test_tiktoken.py +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/bindings/python/conftest.py +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/bindings/python/examples/custom_components.py +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/bindings/python/examples/example.py +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/bindings/python/examples/train_bert_wordpiece.py +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/bindings/python/examples/train_bytelevel_bpe.py +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/bindings/python/examples/train_with_datasets.py +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/bindings/python/examples/using_the_visualizer.ipynb +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/bindings/python/py_src/tokenizers/__init__.py +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/bindings/python/py_src/tokenizers/decoders/__init__.py +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/bindings/python/py_src/tokenizers/implementations/__init__.py +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/bindings/python/py_src/tokenizers/implementations/bert_wordpiece.py +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/bindings/python/py_src/tokenizers/implementations/byte_level_bpe.py +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/bindings/python/py_src/tokenizers/implementations/char_level_bpe.py +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/bindings/python/py_src/tokenizers/implementations/sentencepiece_bpe.py +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/bindings/python/py_src/tokenizers/implementations/sentencepiece_unigram.py +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/bindings/python/py_src/tokenizers/models/__init__.py +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/bindings/python/py_src/tokenizers/models/__init__.pyi +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/bindings/python/py_src/tokenizers/normalizers/__init__.py +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/bindings/python/py_src/tokenizers/normalizers/__init__.pyi +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/bindings/python/py_src/tokenizers/pre_tokenizers/__init__.py +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/bindings/python/py_src/tokenizers/pre_tokenizers/__init__.pyi +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/bindings/python/py_src/tokenizers/processors/__init__.py +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/bindings/python/py_src/tokenizers/processors/__init__.pyi +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/bindings/python/py_src/tokenizers/tools/__init__.py +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/bindings/python/py_src/tokenizers/tools/visualizer-styles.css +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/bindings/python/py_src/tokenizers/trainers/__init__.py +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/bindings/python/rust-toolchain +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/bindings/python/scripts/convert.py +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/bindings/python/scripts/sentencepiece_extractor.py +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/bindings/python/scripts/spm_parity_check.py +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/bindings/python/setup.cfg +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/bindings/python/src/encoding.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/bindings/python/src/error.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/bindings/python/src/models.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/bindings/python/src/normalizers.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/bindings/python/src/pre_tokenizers.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/bindings/python/src/token.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/bindings/python/src/utils/iterators.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/bindings/python/src/utils/mod.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/bindings/python/src/utils/normalization.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/bindings/python/src/utils/pretokenization.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/bindings/python/src/utils/regex.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/bindings/python/src/utils/serde_pyo3.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/bindings/python/stub.py +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/bindings/python/test.txt +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/bindings/python/tests/__init__.py +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/bindings/python/tests/bindings/__init__.py +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/bindings/python/tests/bindings/test_decoders.py +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/bindings/python/tests/bindings/test_encoding.py +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/bindings/python/tests/bindings/test_models.py +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/bindings/python/tests/bindings/test_normalizers.py +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/bindings/python/tests/bindings/test_pre_tokenizers.py +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/bindings/python/tests/bindings/test_processors.py +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/bindings/python/tests/bindings/test_trainers.py +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/bindings/python/tests/documentation/__init__.py +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/bindings/python/tests/documentation/test_pipeline.py +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/bindings/python/tests/documentation/test_quicktour.py +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/bindings/python/tests/documentation/test_tutorial_train_from_iterators.py +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/bindings/python/tests/implementations/__init__.py +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/bindings/python/tests/implementations/test_base_tokenizer.py +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/bindings/python/tests/implementations/test_bert_wordpiece.py +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/bindings/python/tests/implementations/test_byte_level_bpe.py +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/bindings/python/tests/implementations/test_char_bpe.py +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/bindings/python/tests/implementations/test_sentencepiece.py +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/bindings/python/tests/test_serialization.py +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/bindings/python/tests/utils.py +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/py_src/tokenizers/__init__.py +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/py_src/tokenizers/decoders/__init__.py +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/py_src/tokenizers/implementations/__init__.py +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/py_src/tokenizers/implementations/bert_wordpiece.py +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/py_src/tokenizers/implementations/byte_level_bpe.py +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/py_src/tokenizers/implementations/char_level_bpe.py +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/py_src/tokenizers/implementations/sentencepiece_bpe.py +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/py_src/tokenizers/implementations/sentencepiece_unigram.py +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/py_src/tokenizers/models/__init__.py +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/py_src/tokenizers/models/__init__.pyi +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/py_src/tokenizers/normalizers/__init__.py +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/py_src/tokenizers/normalizers/__init__.pyi +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/py_src/tokenizers/pre_tokenizers/__init__.py +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/py_src/tokenizers/pre_tokenizers/__init__.pyi +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/py_src/tokenizers/processors/__init__.py +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/py_src/tokenizers/processors/__init__.pyi +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/py_src/tokenizers/tools/__init__.py +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/py_src/tokenizers/tools/visualizer-styles.css +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/py_src/tokenizers/trainers/__init__.py +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/CHANGELOG.md +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/LICENSE +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/Makefile +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/README.md +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/README.tpl +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/benches/bert_benchmark.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/benches/bpe_benchmark.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/benches/common/mod.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/benches/layout_benchmark.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/benches/llama3_benchmark.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/benches/unigram_benchmark.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/examples/encode_batch.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/examples/serialization.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/src/decoders/bpe.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/src/decoders/byte_fallback.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/src/decoders/ctc.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/src/decoders/fuse.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/src/decoders/mod.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/src/decoders/sequence.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/src/decoders/strip.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/src/decoders/wordpiece.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/src/lib.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/src/models/bpe/mod.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/src/models/bpe/model.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/src/models/bpe/serialization.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/src/models/bpe/word.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/src/models/mod.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/src/models/unigram/lattice.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/src/models/unigram/mod.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/src/models/unigram/serialization.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/src/models/unigram/trainer.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/src/models/wordlevel/mod.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/src/models/wordlevel/serialization.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/src/models/wordlevel/trainer.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/src/models/wordpiece/serialization.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/src/models/wordpiece/trainer.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/src/normalizers/bert.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/src/normalizers/byte_level.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/src/normalizers/mod.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/src/normalizers/precompiled.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/src/normalizers/prepend.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/src/normalizers/replace.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/src/normalizers/strip.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/src/normalizers/unicode.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/src/normalizers/utils.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/src/pre_tokenizers/bert.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/src/pre_tokenizers/byte_level.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/src/pre_tokenizers/delimiter.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/src/pre_tokenizers/digits.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/src/pre_tokenizers/fixed_length.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/src/pre_tokenizers/metaspace.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/src/pre_tokenizers/mod.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/src/pre_tokenizers/punctuation.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/src/pre_tokenizers/sequence.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/src/pre_tokenizers/split.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/src/pre_tokenizers/unicode_scripts/mod.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/src/pre_tokenizers/unicode_scripts/pre_tokenizer.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/src/pre_tokenizers/unicode_scripts/scripts.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/src/pre_tokenizers/whitespace.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/src/processors/bert.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/src/processors/mod.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/src/processors/roberta.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/src/processors/sequence.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/src/processors/template.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/src/tokenizer/added_vocabulary.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/src/tokenizer/encoding.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/src/tokenizer/normalizer.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/src/tokenizer/pattern.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/src/tokenizer/pre_tokenizer.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/src/tokenizer/serialization.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/src/utils/cache.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/src/utils/fancy.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/src/utils/from_pretrained.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/src/utils/iter.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/src/utils/mod.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/src/utils/onig.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/src/utils/padding.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/src/utils/parallelism.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/src/utils/progress.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/src/utils/truncation.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/tests/added_tokens.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/tests/common/mod.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/tests/documentation.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/tests/from_pretrained.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/tests/offsets.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/tests/serialization.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/tests/stream.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/tests/training.rs +0 -0
- {tokenizers-0.21.4.dev0 → tokenizers-0.22.1}/tokenizers/tests/unigram.rs +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: tokenizers
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.22.1
|
|
4
4
|
Classifier: Development Status :: 5 - Production/Stable
|
|
5
5
|
Classifier: Intended Audience :: Developers
|
|
6
6
|
Classifier: Intended Audience :: Education
|
|
@@ -15,8 +15,9 @@ Classifier: Programming Language :: Python :: 3.12
|
|
|
15
15
|
Classifier: Programming Language :: Python :: 3.13
|
|
16
16
|
Classifier: Programming Language :: Python :: 3 :: Only
|
|
17
17
|
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
18
|
-
Requires-Dist: huggingface-hub>=0.16.4,<
|
|
18
|
+
Requires-Dist: huggingface-hub>=0.16.4,<2.0
|
|
19
19
|
Requires-Dist: pytest ; extra == 'testing'
|
|
20
|
+
Requires-Dist: pytest-asyncio ; extra == 'testing'
|
|
20
21
|
Requires-Dist: requests ; extra == 'testing'
|
|
21
22
|
Requires-Dist: numpy ; extra == 'testing'
|
|
22
23
|
Requires-Dist: datasets ; extra == 'testing'
|
|
@@ -30,7 +31,6 @@ Provides-Extra: testing
|
|
|
30
31
|
Provides-Extra: docs
|
|
31
32
|
Provides-Extra: dev
|
|
32
33
|
Keywords: NLP,tokenizer,BPE,transformer,deep learning
|
|
33
|
-
Author: Anthony MOI <m.anthony.moi@gmail.com>
|
|
34
34
|
Author-email: Nicolas Patry <patry.nicolas@protonmail.com>, Anthony Moi <anthony@huggingface.co>
|
|
35
35
|
Requires-Python: >=3.9
|
|
36
36
|
Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
|