tokenizers 0.23.2__tar.gz → 0.23.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {tokenizers-0.23.2 → tokenizers-0.23.3}/PKG-INFO +2 -2
- {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/Cargo.lock +1 -1
- {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/Cargo.toml +1 -1
- {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/Makefile +3 -1
- {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/src/models.rs +2 -13
- {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/src/trainers.rs +1 -1
- {tokenizers-0.23.2 → tokenizers-0.23.3}/py_src/tokenizers/__init__.pyi +7 -7
- tokenizers-0.23.2/py_src/tokenizers/decoders.pyi → tokenizers-0.23.3/py_src/tokenizers/decoders/__init__.pyi +1 -1
- tokenizers-0.23.2/py_src/tokenizers/normalizers.pyi → tokenizers-0.23.3/py_src/tokenizers/normalizers/__init__.pyi +8 -8
- tokenizers-0.23.2/py_src/tokenizers/pre_tokenizers.pyi → tokenizers-0.23.3/py_src/tokenizers/pre_tokenizers/__init__.pyi +4 -4
- tokenizers-0.23.2/py_src/tokenizers/processors.pyi → tokenizers-0.23.3/py_src/tokenizers/processors/__init__.pyi +2 -2
- tokenizers-0.23.2/py_src/tokenizers/trainers.pyi → tokenizers-0.23.3/py_src/tokenizers/trainers/__init__.pyi +161 -14
- {tokenizers-0.23.2 → tokenizers-0.23.3}/pyproject.toml +1 -1
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/tokenizer/mod.rs +60 -34
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/tokenizer/pre_tokenizer.rs +0 -62
- tokenizers-0.23.2/py_src/tokenizers/trainers/__init__.pyi +0 -712
- {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/.cargo/config.toml +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/.gitignore +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/CHANGELOG.md +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/MANIFEST.in +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/README.md +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/benches/test_tiktoken.py +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/conftest.py +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/docs/pyo3.md +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/examples/train_parity_bpe.py +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/pytest.ini +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/rust-toolchain +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/scripts/convert.py +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/scripts/sentencepiece_extractor.py +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/scripts/spm_parity_check.py +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/setup.cfg +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/src/decoders.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/src/encoding.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/src/error.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/src/lib.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/src/normalizers.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/src/pre_tokenizers.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/src/processors.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/src/token.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/src/tokenizer.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/src/utils/iterators.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/src/utils/mod.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/src/utils/normalization.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/src/utils/pretokenization.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/src/utils/regex.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/src/utils/serde_pyo3.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/test.txt +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/tests/__init__.py +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/tests/bindings/__init__.py +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/tests/bindings/test_decoders.py +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/tests/bindings/test_encoding.py +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/tests/bindings/test_models.py +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/tests/bindings/test_normalizers.py +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/tests/bindings/test_pre_tokenizers.py +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/tests/bindings/test_processors.py +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/tests/bindings/test_tokenizer.py +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/tests/bindings/test_trainers.py +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/tests/documentation/__init__.py +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/tests/documentation/test_pipeline.py +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/tests/documentation/test_quicktour.py +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/tests/documentation/test_tutorial_train_from_iterators.py +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/tests/implementations/__init__.py +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/tests/implementations/test_base_tokenizer.py +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/tests/implementations/test_bert_wordpiece.py +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/tests/implementations/test_byte_level_bpe.py +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/tests/implementations/test_char_bpe.py +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/tests/implementations/test_sentencepiece.py +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/tests/test_benchmarks.py +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/tests/test_freethreaded.py +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/tests/test_serialization.py +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/bindings/python/tests/utils.py +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/py_src/tokenizers/__init__.py +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/py_src/tokenizers/decoders/__init__.py +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/py_src/tokenizers/implementations/__init__.py +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/py_src/tokenizers/implementations/base_tokenizer.py +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/py_src/tokenizers/implementations/bert_wordpiece.py +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/py_src/tokenizers/implementations/byte_level_bpe.py +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/py_src/tokenizers/implementations/char_level_bpe.py +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/py_src/tokenizers/implementations/sentencepiece_bpe.py +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/py_src/tokenizers/implementations/sentencepiece_unigram.py +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/py_src/tokenizers/models/__init__.py +0 -0
- /tokenizers-0.23.2/py_src/tokenizers/models.pyi → /tokenizers-0.23.3/py_src/tokenizers/models/__init__.pyi +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/py_src/tokenizers/normalizers/__init__.py +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/py_src/tokenizers/pre_tokenizers/__init__.py +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/py_src/tokenizers/processors/__init__.py +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/py_src/tokenizers/py.typed +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/py_src/tokenizers/tokenizers.pyi +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/py_src/tokenizers/tools/__init__.py +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/py_src/tokenizers/tools/visualizer-styles.css +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/py_src/tokenizers/tools/visualizer.py +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/py_src/tokenizers/trainers/__init__.py +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/CHANGELOG.md +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/Cargo.toml +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/LICENSE +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/Makefile +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/README.md +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/README.tpl +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/benches/added_vocab_deserialize.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/benches/bert_benchmark.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/benches/bpe_benchmark.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/benches/ci_benchmark.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/benches/common/mod.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/benches/decode_benchmark.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/benches/layout_benchmark.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/benches/llama3_benchmark.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/benches/truncation_benchmark.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/benches/unigram_benchmark.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/decoders/bpe.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/decoders/byte_fallback.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/decoders/ctc.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/decoders/fuse.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/decoders/mod.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/decoders/sequence.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/decoders/strip.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/decoders/wordpiece.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/lib.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/models/bpe/mod.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/models/bpe/model.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/models/bpe/parity_trainer.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/models/bpe/serialization.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/models/bpe/trainer.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/models/bpe/word.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/models/mod.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/models/unigram/lattice.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/models/unigram/mod.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/models/unigram/model.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/models/unigram/serialization.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/models/unigram/trainer.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/models/unigram/trie.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/models/wordlevel/mod.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/models/wordlevel/serialization.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/models/wordlevel/trainer.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/models/wordpiece/mod.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/models/wordpiece/serialization.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/models/wordpiece/trainer.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/normalizers/bert.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/normalizers/byte_level.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/normalizers/mod.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/normalizers/precompiled.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/normalizers/prepend.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/normalizers/replace.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/normalizers/strip.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/normalizers/unicode.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/normalizers/utils.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/pre_tokenizers/bert.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/pre_tokenizers/byte_level.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/pre_tokenizers/delimiter.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/pre_tokenizers/digits.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/pre_tokenizers/fixed_length.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/pre_tokenizers/metaspace.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/pre_tokenizers/mod.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/pre_tokenizers/punctuation.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/pre_tokenizers/sequence.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/pre_tokenizers/split.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/pre_tokenizers/unicode_scripts/mod.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/pre_tokenizers/unicode_scripts/pre_tokenizer.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/pre_tokenizers/unicode_scripts/scripts.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/pre_tokenizers/whitespace.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/processors/bert.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/processors/mod.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/processors/roberta.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/processors/sequence.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/processors/template.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/tokenizer/added_vocabulary.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/tokenizer/encoding.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/tokenizer/normalizer.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/tokenizer/pattern.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/tokenizer/serialization.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/utils/cache.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/utils/fancy.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/utils/from_pretrained.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/utils/iter.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/utils/mod.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/utils/onig.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/utils/padding.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/utils/parallelism.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/utils/progress.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/src/utils/truncation.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/tests/added_tokens.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/tests/common/mod.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/tests/documentation.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/tests/from_pretrained.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/tests/offsets.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/tests/serialization.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/tests/stream.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/tests/training.rs +0 -0
- {tokenizers-0.23.2 → tokenizers-0.23.3}/tokenizers/tests/unigram.rs +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: tokenizers
|
|
3
|
-
Version: 0.23.
|
|
3
|
+
Version: 0.23.3
|
|
4
4
|
Classifier: Development Status :: 5 - Production/Stable
|
|
5
5
|
Classifier: Intended Audience :: Developers
|
|
6
6
|
Classifier: Intended Audience :: Education
|
|
@@ -15,7 +15,7 @@ Classifier: Programming Language :: Python :: 3.13
|
|
|
15
15
|
Classifier: Programming Language :: Python :: 3.14
|
|
16
16
|
Classifier: Programming Language :: Python :: 3 :: Only
|
|
17
17
|
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
18
|
-
Requires-Dist: huggingface-hub>=0.16.4,<
|
|
18
|
+
Requires-Dist: huggingface-hub>=0.16.4,<3.0
|
|
19
19
|
Requires-Dist: tokenizers[testing] ; extra == 'dev'
|
|
20
20
|
Requires-Dist: sphinx ; extra == 'docs'
|
|
21
21
|
Requires-Dist: sphinx-rtd-theme ; extra == 'docs'
|
|
@@ -34,13 +34,15 @@ check-style:
|
|
|
34
34
|
$(CARGO_ENV) cargo run --manifest-path ./tools/stub-gen/Cargo.toml
|
|
35
35
|
ruff check $(check_stubs) --select I --fix
|
|
36
36
|
ruff check $(check_dirs)
|
|
37
|
-
ruff format
|
|
37
|
+
ruff format $(check_stubs)
|
|
38
38
|
ruff format --check $(check_dirs)
|
|
39
39
|
ty check py_src --exclude py_src/tokenizers/implementations --exclude py_src/tokenizers/tools/visualizer.py
|
|
40
40
|
|
|
41
41
|
|
|
42
42
|
TESTS_RESOURCES = $(DATA_DIR)/small.txt $(DATA_DIR)/roberta.json
|
|
43
43
|
|
|
44
|
+
test-data: $(TESTS_RESOURCES)
|
|
45
|
+
|
|
44
46
|
# Launch the Python test suite
|
|
45
47
|
test-py: $(TESTS_RESOURCES)
|
|
46
48
|
$(PIP) install pytest pytest-asyncio huggingface_hub setuptools_rust numpy pyarrow datasets
|
|
@@ -53,20 +53,9 @@ impl Model for PyModel {
|
|
|
53
53
|
}
|
|
54
54
|
|
|
55
55
|
/// See [`Model::tokenize_in_pretokenized`] for the lock-once rationale.
|
|
56
|
-
fn tokenize_in_pretokenized(
|
|
57
|
-
&self,
|
|
58
|
-
pretokenized: &mut PreTokenizedString,
|
|
59
|
-
truncation: Option<(usize, tk::TruncationDirection)>,
|
|
60
|
-
) -> tk::Result<()> {
|
|
56
|
+
fn tokenize_in_pretokenized(&self, pretokenized: &mut PreTokenizedString) -> tk::Result<()> {
|
|
61
57
|
let guard = self.model.read().unwrap();
|
|
62
|
-
|
|
63
|
-
Some((max_tokens, direction)) => pretokenized.tokenize_with_limit(
|
|
64
|
-
|normalized| guard.tokenize(normalized.get()),
|
|
65
|
-
max_tokens,
|
|
66
|
-
direction,
|
|
67
|
-
),
|
|
68
|
-
None => pretokenized.tokenize(|normalized| guard.tokenize(normalized.get())),
|
|
69
|
-
}
|
|
58
|
+
pretokenized.tokenize(|normalized| guard.tokenize(normalized.get()))
|
|
70
59
|
}
|
|
71
60
|
|
|
72
61
|
fn token_to_id(&self, token: &str) -> Option<u32> {
|
|
@@ -361,7 +361,7 @@ impl PyBpeTrainer {
|
|
|
361
361
|
#[new]
|
|
362
362
|
#[pyo3(
|
|
363
363
|
signature = (**kwargs),
|
|
364
|
-
text_signature = "(self, vocab_size=30000, min_frequency=0, show_progress=True, progress_format=\"indicatif\", special_tokens=[], limit_alphabet=None, initial_alphabet=[], continuing_subword_prefix=None, end_of_word_suffix=None, max_token_length=None
|
|
364
|
+
text_signature = "(self, vocab_size=30000, min_frequency=0, show_progress=True, progress_format=\"indicatif\", special_tokens=[], limit_alphabet=None, initial_alphabet=[], continuing_subword_prefix=None, end_of_word_suffix=None, max_token_length=None)"
|
|
365
365
|
)]
|
|
366
366
|
pub fn new(kwargs: Option<&Bound<'_, PyDict>>) -> PyResult<PyClassInitializer<Self>> {
|
|
367
367
|
let mut builder = tk::models::bpe::BpeTrainer::builder();
|
|
@@ -51,14 +51,14 @@ class AddedToken:
|
|
|
51
51
|
special (:obj:`bool`, defaults to :obj:`False` with :meth:`~tokenizers.Tokenizer.add_tokens` and :obj:`False` with :meth:`~tokenizers.Tokenizer.add_special_tokens`):
|
|
52
52
|
Defines whether this token should be skipped when decoding.
|
|
53
53
|
"""
|
|
54
|
-
def __eq__(self,
|
|
55
|
-
def __ge__(self,
|
|
54
|
+
def __eq__(self, other: object, /) -> bool: ...
|
|
55
|
+
def __ge__(self, other: object, /) -> bool: ...
|
|
56
56
|
def __getstate__(self, /) -> dict: ...
|
|
57
|
-
def __gt__(self,
|
|
57
|
+
def __gt__(self, other: object, /) -> bool: ...
|
|
58
58
|
def __hash__(self, /) -> int: ...
|
|
59
|
-
def __le__(self,
|
|
60
|
-
def __lt__(self,
|
|
61
|
-
def __ne__(self,
|
|
59
|
+
def __le__(self, other: object, /) -> bool: ...
|
|
60
|
+
def __lt__(self, other: object, /) -> bool: ...
|
|
61
|
+
def __ne__(self, other: object, /) -> bool: ...
|
|
62
62
|
def __new__(cls, /, content: str | None = None, **kwargs) -> AddedToken: ...
|
|
63
63
|
def __repr__(self, /) -> str: ...
|
|
64
64
|
def __setstate__(self, /, state: Any) -> None: ...
|
|
@@ -435,7 +435,7 @@ class NormalizedString:
|
|
|
435
435
|
sequence: str:
|
|
436
436
|
The string sequence used to initialize this NormalizedString
|
|
437
437
|
"""
|
|
438
|
-
def __getitem__(self,
|
|
438
|
+
def __getitem__(self, range: int | tuple[int, int] | slice, /) -> NormalizedString | None: ...
|
|
439
439
|
def __new__(cls, /, sequence: str) -> NormalizedString: ...
|
|
440
440
|
def __repr__(self, /) -> str: ...
|
|
441
441
|
def __str__(self, /) -> str: ...
|
|
@@ -80,7 +80,7 @@ class ByteLevel(Normalizer):
|
|
|
80
80
|
|
|
81
81
|
>>> from tokenizers.normalizers import ByteLevel
|
|
82
82
|
>>> normalizer = ByteLevel()
|
|
83
|
-
>>> normalizer.normalize_str("hello
|
|
83
|
+
>>> normalizer.normalize_str("hello\\nworld")
|
|
84
84
|
'helloĊworld'
|
|
85
85
|
"""
|
|
86
86
|
def __new__(cls, /) -> ByteLevel: ...
|
|
@@ -115,7 +115,7 @@ class NFC(Normalizer):
|
|
|
115
115
|
|
|
116
116
|
>>> from tokenizers.normalizers import NFC
|
|
117
117
|
>>> normalizer = NFC()
|
|
118
|
-
>>> normalizer.normalize_str("e
|
|
118
|
+
>>> normalizer.normalize_str("e\\u0301") # 'e' + combining accent
|
|
119
119
|
'é'
|
|
120
120
|
"""
|
|
121
121
|
def __new__(cls, /) -> NFC: ...
|
|
@@ -137,7 +137,7 @@ class NFD(Normalizer):
|
|
|
137
137
|
>>> from tokenizers.normalizers import NFD
|
|
138
138
|
>>> normalizer = NFD()
|
|
139
139
|
>>> normalizer.normalize_str("Héllo")
|
|
140
|
-
'He
|
|
140
|
+
'He\\u0301llo'
|
|
141
141
|
"""
|
|
142
142
|
def __new__(cls, /) -> NFD: ...
|
|
143
143
|
|
|
@@ -155,7 +155,7 @@ class NFKC(Normalizer):
|
|
|
155
155
|
|
|
156
156
|
>>> from tokenizers.normalizers import NFKC
|
|
157
157
|
>>> normalizer = NFKC()
|
|
158
|
-
>>> normalizer.normalize_str("fine caf
|
|
158
|
+
>>> normalizer.normalize_str("fine caf\\u00e9")
|
|
159
159
|
'fine café'
|
|
160
160
|
"""
|
|
161
161
|
def __new__(cls, /) -> NFKC: ...
|
|
@@ -192,7 +192,7 @@ class Nmt(Normalizer):
|
|
|
192
192
|
|
|
193
193
|
>>> from tokenizers.normalizers import Nmt
|
|
194
194
|
>>> normalizer = Nmt()
|
|
195
|
-
>>> normalizer.normalize_str("Hello
|
|
195
|
+
>>> normalizer.normalize_str("Hello\\x00World")
|
|
196
196
|
'Hello World'
|
|
197
197
|
"""
|
|
198
198
|
def __new__(cls, /) -> Nmt: ...
|
|
@@ -310,7 +310,7 @@ class Replace(Normalizer):
|
|
|
310
310
|
>>> Replace(".", " ").normalize_str("hello.world")
|
|
311
311
|
'hello world'
|
|
312
312
|
>>> # Replace using a regex
|
|
313
|
-
>>> Replace(Regex(r"
|
|
313
|
+
>>> Replace(Regex(r"\\s+"), " ").normalize_str("hello world")
|
|
314
314
|
'hello world'
|
|
315
315
|
"""
|
|
316
316
|
def __new__(cls, /, pattern: str | Regex, content: str) -> Replace: ...
|
|
@@ -340,11 +340,11 @@ class Sequence(Normalizer):
|
|
|
340
340
|
>>> normalizer.normalize_str("Héllo Wörld")
|
|
341
341
|
'hello world'
|
|
342
342
|
"""
|
|
343
|
-
def __getitem__(self,
|
|
343
|
+
def __getitem__(self, index: int, /) -> Any: ...
|
|
344
344
|
def __getnewargs__(self, /) -> tuple: ...
|
|
345
345
|
def __len__(self, /) -> int: ...
|
|
346
346
|
def __new__(cls, /, normalizers: list) -> Sequence: ...
|
|
347
|
-
def __setitem__(self,
|
|
347
|
+
def __setitem__(self, index: int, value: Any, /) -> None: ...
|
|
348
348
|
|
|
349
349
|
@final
|
|
350
350
|
class Strip(Normalizer):
|
|
@@ -279,10 +279,10 @@ class Sequence(PreTokenizer):
|
|
|
279
279
|
>>> pre_tokenizer.pre_tokenize_str("Hello, world!")
|
|
280
280
|
[('Hello', (0, 5)), (',', (5, 6)), ('world', (7, 12)), ('!', (12, 13))]
|
|
281
281
|
"""
|
|
282
|
-
def __getitem__(self,
|
|
282
|
+
def __getitem__(self, index: int, /) -> Any: ...
|
|
283
283
|
def __getnewargs__(self, /) -> tuple: ...
|
|
284
284
|
def __new__(cls, /, pre_tokenizers: list) -> Sequence: ...
|
|
285
|
-
def __setitem__(self,
|
|
285
|
+
def __setitem__(self, index: int, value: Any, /) -> None: ...
|
|
286
286
|
|
|
287
287
|
@final
|
|
288
288
|
class Split(PreTokenizer):
|
|
@@ -317,7 +317,7 @@ class Split(PreTokenizer):
|
|
|
317
317
|
>>> pre_tokenizer.pre_tokenize_str("one,two,three")
|
|
318
318
|
[('one', (0, 3)), ('two', (4, 7)), ('three', (8, 13))]
|
|
319
319
|
>>> # Split using a regex, keeping the delimiter isolated
|
|
320
|
-
>>> Split(Regex(r"
|
|
320
|
+
>>> Split(Regex(r"\\s+"), behavior="isolated").pre_tokenize_str("hello world")
|
|
321
321
|
[('hello', (0, 5)), (' ', (5, 8)), ('world', (8, 13))]
|
|
322
322
|
"""
|
|
323
323
|
def __getnewargs__(self, /) -> tuple: ...
|
|
@@ -355,7 +355,7 @@ class UnicodeScripts(PreTokenizer):
|
|
|
355
355
|
@final
|
|
356
356
|
class Whitespace(PreTokenizer):
|
|
357
357
|
"""
|
|
358
|
-
This pre-tokenizer splits on word boundaries according to the
|
|
358
|
+
This pre-tokenizer splits on word boundaries according to the ``\\w+|[^\\w\\s]+``
|
|
359
359
|
regex pattern. It splits on word characters or characters that aren't words or
|
|
360
360
|
whitespaces (punctuation such as hyphens, apostrophes, commas, etc.).
|
|
361
361
|
|
|
@@ -207,10 +207,10 @@ class Sequence(PostProcessor):
|
|
|
207
207
|
>>> from tokenizers.processors import BertProcessing, ByteLevel, Sequence
|
|
208
208
|
>>> processor = Sequence([ByteLevel(trim_offsets=True), BertProcessing(("[SEP]", 102), ("[CLS]", 101))])
|
|
209
209
|
"""
|
|
210
|
-
def __getitem__(self,
|
|
210
|
+
def __getitem__(self, index: int, /) -> Any: ...
|
|
211
211
|
def __getnewargs__(self, /) -> tuple: ...
|
|
212
212
|
def __new__(cls, /, processors_py: list) -> Sequence: ...
|
|
213
|
-
def __setitem__(self,
|
|
213
|
+
def __setitem__(self, index: int, value: Any, /) -> None: ...
|
|
214
214
|
|
|
215
215
|
@final
|
|
216
216
|
class TemplateProcessing(PostProcessor):
|
|
@@ -5,7 +5,7 @@ Trainers Module
|
|
|
5
5
|
from collections.abc import Sequence
|
|
6
6
|
from typing import Any, final
|
|
7
7
|
|
|
8
|
-
from tokenizers import AddedToken
|
|
8
|
+
from tokenizers import AddedToken, Tokenizer
|
|
9
9
|
|
|
10
10
|
@final
|
|
11
11
|
class BpeTrainer(Trainer):
|
|
@@ -57,7 +57,21 @@ class BpeTrainer(Trainer):
|
|
|
57
57
|
>>> tokenizer = Tokenizer(BPE())
|
|
58
58
|
>>> tokenizer.train(["path/to/corpus.txt"], trainer)
|
|
59
59
|
"""
|
|
60
|
-
def __new__(
|
|
60
|
+
def __new__(
|
|
61
|
+
cls,
|
|
62
|
+
/,
|
|
63
|
+
*,
|
|
64
|
+
vocab_size: int = 30000,
|
|
65
|
+
min_frequency: int = 0,
|
|
66
|
+
show_progress: bool = True,
|
|
67
|
+
progress_format: str = "indicatif",
|
|
68
|
+
special_tokens: list = [],
|
|
69
|
+
limit_alphabet: int | None = None,
|
|
70
|
+
initial_alphabet: Sequence[str] = [],
|
|
71
|
+
continuing_subword_prefix: str | None = None,
|
|
72
|
+
end_of_word_suffix: str | None = None,
|
|
73
|
+
max_token_length: int | None = None,
|
|
74
|
+
) -> BpeTrainer: ...
|
|
61
75
|
@property
|
|
62
76
|
def continuing_subword_prefix(self, /) -> str | None: ...
|
|
63
77
|
@continuing_subword_prefix.setter
|
|
@@ -111,6 +125,75 @@ class BpeTrainer(Trainer):
|
|
|
111
125
|
|
|
112
126
|
@final
|
|
113
127
|
class ParityBpeTrainer:
|
|
128
|
+
"""
|
|
129
|
+
Trainer for parity-aware BPE that ensures cross-lingual fairness in tokenization.
|
|
130
|
+
|
|
131
|
+
Unlike standard BPE, this trainer takes one Python iterator per language and
|
|
132
|
+
balances merge operations across languages using a development set or target
|
|
133
|
+
compression ratios. The single training entry point is
|
|
134
|
+
:meth:`train_from_iterator`, the multi-corpus analogue of
|
|
135
|
+
:meth:`tokenizers.Tokenizer.train_from_iterator`. Balancing requires either
|
|
136
|
+
``dev_iterators`` (ideally parallel across languages) or ``ratio``; given
|
|
137
|
+
neither, selection follows the per-language training totals, which the
|
|
138
|
+
highest-data language dominates, so the result stays close to plain BPE. When
|
|
139
|
+
both are given the dev set takes precedence. A runnable version that obtains
|
|
140
|
+
per-language corpora and a parallel dev set is in
|
|
141
|
+
``examples/train_parity_bpe.py``.
|
|
142
|
+
|
|
143
|
+
Args:
|
|
144
|
+
num_merges (:obj:`int`, `optional`):
|
|
145
|
+
Number of BPE merge operations to perform. Defaults to ``32000``.
|
|
146
|
+
|
|
147
|
+
variant (:obj:`str`, `optional`):
|
|
148
|
+
Algorithm variant: ``"base"`` (default) or ``"window"`` (moving-window balancing).
|
|
149
|
+
|
|
150
|
+
min_frequency (:obj:`int`, `optional`):
|
|
151
|
+
Minimum pair frequency to merge. Defaults to ``0``.
|
|
152
|
+
|
|
153
|
+
ratio (:obj:`List[float]`, `optional`):
|
|
154
|
+
Target compression rate per language, one entry per training iterator. The
|
|
155
|
+
trainer selects the language with the lowest ``compression_rate / ratio``,
|
|
156
|
+
so raising one language's ratio gives it more merges; only the values
|
|
157
|
+
relative to each other matter. Rates are counted in the units the
|
|
158
|
+
pre-tokenizer emits, bytes under
|
|
159
|
+
:class:`~tokenizers.pre_tokenizers.ByteLevel`, so equal ratios do not give
|
|
160
|
+
equal tokenization across scripts: Devanagari takes about 2.5x the bytes of
|
|
161
|
+
Latin script for the same content. Set each ratio proportional to the
|
|
162
|
+
language's average length on parallel text instead. Defaults to ``None``.
|
|
163
|
+
|
|
164
|
+
global_merges (:obj:`int`, `optional`):
|
|
165
|
+
Number of initial standard BPE merges before switching to parity mode. Defaults to ``0``.
|
|
166
|
+
|
|
167
|
+
window_size (:obj:`int`, `optional`):
|
|
168
|
+
Window size for the ``"window"`` variant. Defaults to ``100``.
|
|
169
|
+
|
|
170
|
+
alpha (:obj:`float`, `optional`):
|
|
171
|
+
Alpha parameter for the ``"window"`` variant. Defaults to ``2.0``.
|
|
172
|
+
|
|
173
|
+
total_symbols (:obj:`bool`, `optional`):
|
|
174
|
+
If True, subtract unique character count from ``num_merges``. Defaults to ``False``.
|
|
175
|
+
|
|
176
|
+
Example::
|
|
177
|
+
|
|
178
|
+
>>> from tokenizers import Tokenizer, pre_tokenizers
|
|
179
|
+
>>> from tokenizers.models import BPE
|
|
180
|
+
>>> from tokenizers.trainers import ParityBpeTrainer
|
|
181
|
+
>>> tokenizer = Tokenizer(BPE())
|
|
182
|
+
>>> tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel()
|
|
183
|
+
>>> trainer = ParityBpeTrainer(num_merges=32000, variant="base")
|
|
184
|
+
>>> # balance against a parallel dev set, the same sentences in every language
|
|
185
|
+
>>> trainer.train_from_iterator(
|
|
186
|
+
... tokenizer,
|
|
187
|
+
... train_iterators=[english_lines, hindi_lines],
|
|
188
|
+
... dev_iterators=[english_dev, hindi_dev],
|
|
189
|
+
... )
|
|
190
|
+
>>> # or against target rates, which need no dev data
|
|
191
|
+
>>> trainer.train_from_iterator(
|
|
192
|
+
... tokenizer,
|
|
193
|
+
... train_iterators=[english_lines, hindi_lines],
|
|
194
|
+
... ratio=[1.0, 2.57],
|
|
195
|
+
... )
|
|
196
|
+
"""
|
|
114
197
|
def __getstate__(self, /) -> Any: ...
|
|
115
198
|
def __new__(
|
|
116
199
|
cls,
|
|
@@ -130,16 +213,10 @@ class ParityBpeTrainer:
|
|
|
130
213
|
continuing_subword_prefix: str | None = None,
|
|
131
214
|
end_of_word_suffix: str | None = None,
|
|
132
215
|
max_token_length: int | None = None,
|
|
133
|
-
) -> ParityBpeTrainer:
|
|
134
|
-
|
|
135
|
-
...
|
|
136
|
-
def __repr__(self, /) -> str:
|
|
137
|
-
"""Return repr(self)."""
|
|
138
|
-
...
|
|
216
|
+
) -> ParityBpeTrainer: ...
|
|
217
|
+
def __repr__(self, /) -> str: ...
|
|
139
218
|
def __setstate__(self, /, state: Any) -> None: ...
|
|
140
|
-
def __str__(self, /) -> str:
|
|
141
|
-
"""Return str(self)."""
|
|
142
|
-
...
|
|
219
|
+
def __str__(self, /) -> str: ...
|
|
143
220
|
@property
|
|
144
221
|
def alpha(self, /) -> float: ...
|
|
145
222
|
@alpha.setter
|
|
@@ -188,6 +265,44 @@ class ParityBpeTrainer:
|
|
|
188
265
|
def total_symbols(self, /) -> bool: ...
|
|
189
266
|
@total_symbols.setter
|
|
190
267
|
def total_symbols(self, /, v: bool) -> None: ...
|
|
268
|
+
def train_from_iterator(
|
|
269
|
+
self,
|
|
270
|
+
/,
|
|
271
|
+
tokenizer: Tokenizer,
|
|
272
|
+
train_iterators: Sequence[Any],
|
|
273
|
+
dev_iterators: Sequence[Any] | None = None,
|
|
274
|
+
ratio: Sequence[float] | None = None,
|
|
275
|
+
) -> None:
|
|
276
|
+
"""
|
|
277
|
+
Train a user-configured tokenizer with parity-aware BPE from per-language
|
|
278
|
+
Python iterators.
|
|
279
|
+
|
|
280
|
+
Each entry of ``train_iterators`` (and optionally ``dev_iterators``) is a
|
|
281
|
+
Python iterator yielding strings (or batches / lists of strings) for one
|
|
282
|
+
language. This is the multi-corpus analogue of
|
|
283
|
+
:meth:`~tokenizers.Tokenizer.train_from_iterator`: file I/O happens in
|
|
284
|
+
Python, so users can pull data from plain text, parquet (via ``pyarrow``),
|
|
285
|
+
``datasets``, etc.
|
|
286
|
+
|
|
287
|
+
Args:
|
|
288
|
+
tokenizer (:class:`~tokenizers.Tokenizer`):
|
|
289
|
+
A tokenizer instance to train. Its pre-tokenizer (and optionally
|
|
290
|
+
normalizer) should already be configured.
|
|
291
|
+
|
|
292
|
+
train_iterators (:obj:`List[Iterator]`):
|
|
293
|
+
One Python iterator per language, each yielding ``str`` or
|
|
294
|
+
``List[str]``.
|
|
295
|
+
|
|
296
|
+
dev_iterators (:obj:`List[Iterator]`, `optional`):
|
|
297
|
+
One Python iterator per language, used to drive parity-aware
|
|
298
|
+
language selection. Must have the same length as
|
|
299
|
+
``train_iterators``.
|
|
300
|
+
|
|
301
|
+
ratio (:obj:`List[float]`, `optional`):
|
|
302
|
+
Target compression rates per language, an alternative to
|
|
303
|
+
``dev_iterators`` and ignored when one is supplied. See the class
|
|
304
|
+
docstring for how to choose the values.
|
|
305
|
+
"""
|
|
191
306
|
@property
|
|
192
307
|
def variant(self, /) -> str: ...
|
|
193
308
|
@property
|
|
@@ -254,7 +369,19 @@ class UnigramTrainer(Trainer):
|
|
|
254
369
|
>>> tokenizer = Tokenizer(Unigram())
|
|
255
370
|
>>> tokenizer.train(["path/to/corpus.txt"], trainer)
|
|
256
371
|
"""
|
|
257
|
-
def __new__(
|
|
372
|
+
def __new__(
|
|
373
|
+
cls,
|
|
374
|
+
/,
|
|
375
|
+
*,
|
|
376
|
+
vocab_size: int = 8000,
|
|
377
|
+
show_progress: bool = True,
|
|
378
|
+
special_tokens: list = [],
|
|
379
|
+
initial_alphabet: Sequence[str] = [],
|
|
380
|
+
shrinking_factor=0.75,
|
|
381
|
+
unk_token=None,
|
|
382
|
+
max_piece_length=16,
|
|
383
|
+
n_sub_iterations=2,
|
|
384
|
+
) -> UnigramTrainer: ...
|
|
258
385
|
@property
|
|
259
386
|
def initial_alphabet(self, /) -> list[str]: ...
|
|
260
387
|
@initial_alphabet.setter
|
|
@@ -302,7 +429,15 @@ class WordLevelTrainer(Trainer):
|
|
|
302
429
|
>>> tokenizer = Tokenizer(WordLevel(unk_token="<unk>"))
|
|
303
430
|
>>> tokenizer.train(["path/to/corpus.txt"], trainer)
|
|
304
431
|
"""
|
|
305
|
-
def __new__(
|
|
432
|
+
def __new__(
|
|
433
|
+
cls,
|
|
434
|
+
/,
|
|
435
|
+
*,
|
|
436
|
+
vocab_size: int = 30000,
|
|
437
|
+
min_frequency: int = 0,
|
|
438
|
+
show_progress: bool = True,
|
|
439
|
+
special_tokens: list = [],
|
|
440
|
+
) -> WordLevelTrainer: ...
|
|
306
441
|
@property
|
|
307
442
|
def min_frequency(self, /) -> int: ...
|
|
308
443
|
@min_frequency.setter
|
|
@@ -364,7 +499,19 @@ class WordPieceTrainer(Trainer):
|
|
|
364
499
|
>>> tokenizer = Tokenizer(WordPiece(unk_token="[UNK]"))
|
|
365
500
|
>>> tokenizer.train(["path/to/corpus.txt"], trainer)
|
|
366
501
|
"""
|
|
367
|
-
def __new__(
|
|
502
|
+
def __new__(
|
|
503
|
+
cls,
|
|
504
|
+
/,
|
|
505
|
+
*,
|
|
506
|
+
vocab_size: int = 30000,
|
|
507
|
+
min_frequency: int = 0,
|
|
508
|
+
show_progress: bool = True,
|
|
509
|
+
special_tokens: list = [],
|
|
510
|
+
limit_alphabet: int | None = None,
|
|
511
|
+
initial_alphabet: Sequence[str] = [],
|
|
512
|
+
continuing_subword_prefix: str | None = "##",
|
|
513
|
+
end_of_word_suffix: str | None = None,
|
|
514
|
+
) -> WordPieceTrainer: ...
|
|
368
515
|
@property
|
|
369
516
|
def continuing_subword_prefix(self, /) -> str | None: ...
|
|
370
517
|
@continuing_subword_prefix.setter
|
|
@@ -23,7 +23,7 @@ classifiers = [
|
|
|
23
23
|
]
|
|
24
24
|
keywords = ["NLP", "tokenizer", "BPE", "transformer", "deep learning"]
|
|
25
25
|
dynamic = ["description", "license", "readme", "version"]
|
|
26
|
-
dependencies = ["huggingface_hub>=0.16.4,<
|
|
26
|
+
dependencies = ["huggingface_hub>=0.16.4,<3.0"]
|
|
27
27
|
|
|
28
28
|
[project.urls]
|
|
29
29
|
Homepage = "https://github.com/huggingface/tokenizers"
|
|
@@ -88,32 +88,15 @@ pub trait Model {
|
|
|
88
88
|
|
|
89
89
|
/// Tokenize every pre-token within a `PreTokenizedString` in one call.
|
|
90
90
|
///
|
|
91
|
-
/// When `truncation` is `Some((max_tokens, direction))`, the
|
|
92
|
-
/// `tokenize_with_limit` early-exit path is taken; otherwise every
|
|
93
|
-
/// pre-token is tokenized.
|
|
94
|
-
///
|
|
95
91
|
/// The default calls `self.tokenize()` per pre-token, which is correct
|
|
96
92
|
/// for self-contained `Model` implementations. Implementations that
|
|
97
93
|
/// wrap their inner model behind a lock (e.g. the `PyModel` and Node
|
|
98
94
|
/// `Model` bindings, both `Arc<RwLock<_>>`) can override this to
|
|
99
95
|
/// acquire the lock once for the whole sequence of pre-tokens; that
|
|
100
96
|
/// removes ~one atomic load/store pair per pre-token from the hot path
|
|
101
|
-
/// (~1 500 pre-tokens per ~10 KB document).
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
fn tokenize_in_pretokenized(
|
|
105
|
-
&self,
|
|
106
|
-
pretokenized: &mut PreTokenizedString,
|
|
107
|
-
truncation: Option<(usize, TruncationDirection)>,
|
|
108
|
-
) -> Result<()> {
|
|
109
|
-
match truncation {
|
|
110
|
-
Some((max_tokens, direction)) => pretokenized.tokenize_with_limit(
|
|
111
|
-
|normalized| self.tokenize(normalized.get()),
|
|
112
|
-
max_tokens,
|
|
113
|
-
direction,
|
|
114
|
-
),
|
|
115
|
-
None => pretokenized.tokenize(|normalized| self.tokenize(normalized.get())),
|
|
116
|
-
}
|
|
97
|
+
/// (~1 500 pre-tokens per ~10 KB document).
|
|
98
|
+
fn tokenize_in_pretokenized(&self, pretokenized: &mut PreTokenizedString) -> Result<()> {
|
|
99
|
+
pretokenized.tokenize(|normalized| self.tokenize(normalized.get()))
|
|
117
100
|
}
|
|
118
101
|
}
|
|
119
102
|
|
|
@@ -1183,19 +1166,7 @@ where
|
|
|
1183
1166
|
offsets_type: OffsetType,
|
|
1184
1167
|
) -> Result<Encoding> {
|
|
1185
1168
|
let mut pretokenized: PreTokenizedString = pretokenized.into();
|
|
1186
|
-
|
|
1187
|
-
Some(TruncationParams {
|
|
1188
|
-
direction,
|
|
1189
|
-
max_length,
|
|
1190
|
-
strategy,
|
|
1191
|
-
..
|
|
1192
|
-
}) if *strategy != TruncationStrategy::OnlySecond || type_id != 0 => {
|
|
1193
|
-
Some((*max_length, *direction))
|
|
1194
|
-
}
|
|
1195
|
-
_ => None,
|
|
1196
|
-
};
|
|
1197
|
-
self.model
|
|
1198
|
-
.tokenize_in_pretokenized(&mut pretokenized, truncation)?;
|
|
1169
|
+
self.model.tokenize_in_pretokenized(&mut pretokenized)?;
|
|
1199
1170
|
pretokenized.into_encoding(word_idx, type_id, offsets_type)
|
|
1200
1171
|
}
|
|
1201
1172
|
}
|
|
@@ -1666,7 +1637,7 @@ mod tests {
|
|
|
1666
1637
|
}
|
|
1667
1638
|
|
|
1668
1639
|
#[test]
|
|
1669
|
-
fn
|
|
1640
|
+
fn right_truncation_matches_full_encode() {
|
|
1670
1641
|
// "a b c d e f g h i j" → 10 tokens [0,1,2,3,4,5,6,7,8,9]
|
|
1671
1642
|
// Right truncation to 3 → [0,1,2]
|
|
1672
1643
|
let input = "a b c d e f g h i j";
|
|
@@ -1692,6 +1663,61 @@ mod tests {
|
|
|
1692
1663
|
);
|
|
1693
1664
|
}
|
|
1694
1665
|
|
|
1666
|
+
fn overflowing_ids(encoding: &Encoding) -> Vec<Vec<u32>> {
|
|
1667
|
+
encoding
|
|
1668
|
+
.get_overflowing()
|
|
1669
|
+
.iter()
|
|
1670
|
+
.map(|o| o.get_ids().to_vec())
|
|
1671
|
+
.collect()
|
|
1672
|
+
}
|
|
1673
|
+
|
|
1674
|
+
#[test]
|
|
1675
|
+
fn truncation_keeps_every_overflowing_chunk() {
|
|
1676
|
+
let input = "a b c d e f g h i j";
|
|
1677
|
+
for direction in [TruncationDirection::Right, TruncationDirection::Left] {
|
|
1678
|
+
for stride in [0, 1] {
|
|
1679
|
+
let mut tok = test_tokenizer();
|
|
1680
|
+
tok.with_truncation(Some(TruncationParams {
|
|
1681
|
+
max_length: 3,
|
|
1682
|
+
strategy: TruncationStrategy::LongestFirst,
|
|
1683
|
+
stride,
|
|
1684
|
+
direction,
|
|
1685
|
+
}))
|
|
1686
|
+
.unwrap();
|
|
1687
|
+
let truncated = tok.encode(input, false).unwrap();
|
|
1688
|
+
|
|
1689
|
+
let mut expected = test_tokenizer().encode(input, false).unwrap();
|
|
1690
|
+
expected.truncate(3, stride, direction);
|
|
1691
|
+
|
|
1692
|
+
assert_eq!(truncated.get_ids(), expected.get_ids());
|
|
1693
|
+
assert_eq!(
|
|
1694
|
+
overflowing_ids(&truncated),
|
|
1695
|
+
overflowing_ids(&expected),
|
|
1696
|
+
"{direction:?}, stride {stride}"
|
|
1697
|
+
);
|
|
1698
|
+
}
|
|
1699
|
+
}
|
|
1700
|
+
}
|
|
1701
|
+
|
|
1702
|
+
#[test]
|
|
1703
|
+
fn right_truncation_overflow_covers_the_whole_input() {
|
|
1704
|
+
let mut tok = test_tokenizer();
|
|
1705
|
+
tok.with_truncation(Some(TruncationParams {
|
|
1706
|
+
max_length: 3,
|
|
1707
|
+
strategy: TruncationStrategy::LongestFirst,
|
|
1708
|
+
stride: 0,
|
|
1709
|
+
direction: TruncationDirection::Right,
|
|
1710
|
+
}))
|
|
1711
|
+
.unwrap();
|
|
1712
|
+
let encoding = tok.encode("a b c d e f g h i j", false).unwrap();
|
|
1713
|
+
|
|
1714
|
+
assert_eq!(encoding.get_ids(), &[0, 1, 2]);
|
|
1715
|
+
assert_eq!(
|
|
1716
|
+
overflowing_ids(&encoding),
|
|
1717
|
+
vec![vec![3, 4, 5], vec![6, 7, 8], vec![9]]
|
|
1718
|
+
);
|
|
1719
|
+
}
|
|
1720
|
+
|
|
1695
1721
|
#[test]
|
|
1696
1722
|
fn left_truncation_keeps_tail_tokens() {
|
|
1697
1723
|
// "a b c d e f g h i j" → 10 tokens [0,1,2,3,4,5,6,7,8,9]
|