tokenicer 0.0.1.dev0__tar.gz → 0.0.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {tokenicer-0.0.1.dev0 → tokenicer-0.0.3}/LICENSE +2 -0
- tokenicer-0.0.3/MANIFEST.in +2 -0
- tokenicer-0.0.3/PKG-INFO +102 -0
- tokenicer-0.0.3/README.md +78 -0
- tokenicer-0.0.3/requirements.txt +1 -0
- tokenicer-0.0.3/setup.py +44 -0
- tokenicer-0.0.3/tokenicer/__init__.py +19 -0
- tokenicer-0.0.3/tokenicer/const.py +36 -0
- tokenicer-0.0.3/tokenicer/tokenicer.py +172 -0
- tokenicer-0.0.3/tokenicer/util.py +50 -0
- tokenicer-0.0.3/tokenicer.egg-info/PKG-INFO +102 -0
- tokenicer-0.0.3/tokenicer.egg-info/SOURCES.txt +14 -0
- tokenicer-0.0.3/tokenicer.egg-info/requires.txt +1 -0
- tokenicer-0.0.1.dev0/PKG-INFO +0 -23
- tokenicer-0.0.1.dev0/README.md +0 -1
- tokenicer-0.0.1.dev0/setup.py +0 -24
- tokenicer-0.0.1.dev0/tokenicer/__init__.py +0 -0
- tokenicer-0.0.1.dev0/tokenicer.egg-info/PKG-INFO +0 -23
- tokenicer-0.0.1.dev0/tokenicer.egg-info/SOURCES.txt +0 -8
- {tokenicer-0.0.1.dev0 → tokenicer-0.0.3}/setup.cfg +0 -0
- {tokenicer-0.0.1.dev0 → tokenicer-0.0.3}/tokenicer.egg-info/dependency_links.txt +0 -0
- {tokenicer-0.0.1.dev0 → tokenicer-0.0.3}/tokenicer.egg-info/top_level.txt +0 -0
tokenicer-0.0.3/PKG-INFO
ADDED
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
Metadata-Version: 2.2
|
|
2
|
+
Name: tokenicer
|
|
3
|
+
Version: 0.0.3
|
|
4
|
+
Summary: A (nicer) tokenizer you want to use for model `inference` and `training`: with all known peventable `gotchas` normalized or auto-fixed.
|
|
5
|
+
Home-page: https://github.com/ModelCloud/Tokenicer
|
|
6
|
+
Author: ModelCloud
|
|
7
|
+
Author-email: qubitium@modelcloud.ai
|
|
8
|
+
Classifier: Programming Language :: Python :: 3
|
|
9
|
+
Classifier: License :: OSI Approved :: Apache Software License
|
|
10
|
+
Classifier: Operating System :: OS Independent
|
|
11
|
+
Requires-Python: >=3
|
|
12
|
+
Description-Content-Type: text/markdown
|
|
13
|
+
License-File: LICENSE
|
|
14
|
+
Requires-Dist: transformers
|
|
15
|
+
Dynamic: author
|
|
16
|
+
Dynamic: author-email
|
|
17
|
+
Dynamic: classifier
|
|
18
|
+
Dynamic: description
|
|
19
|
+
Dynamic: description-content-type
|
|
20
|
+
Dynamic: home-page
|
|
21
|
+
Dynamic: requires-dist
|
|
22
|
+
Dynamic: requires-python
|
|
23
|
+
Dynamic: summary
|
|
24
|
+
|
|
25
|
+
<h1 align="center">Toke(n)icer</h1>
|
|
26
|
+
<p align="center">A (nicer) tokenizer you want to use for model inference and training: with all known peventable gotchas normalized or auto-fixed.</p>
|
|
27
|
+
<p align="center">
|
|
28
|
+
<a href="https://github.com/ModelCloud/Tokenicer/releases" style="text-decoration:none;"><img alt="GitHub release" src="https://img.shields.io/github/release/ModelCloud/Tokenicer.svg"></a>
|
|
29
|
+
<a href="https://pypi.org/project/tokenicer/" style="text-decoration:none;"><img alt="PyPI - Version" src="https://img.shields.io/pypi/v/tokenicer"></a>
|
|
30
|
+
<a href="https://pepy.tech/projects/tokenicer" style="text-decoration:none;"><img src="https://static.pepy.tech/badge/tokenicer" alt="PyPI Downloads"></a>
|
|
31
|
+
<a href="https://github.com/ModelCloud/tokenicer/blob/main/LICENSE"><img src="https://img.shields.io/pypi/l/tokenicer"></a>
|
|
32
|
+
<a href="https://huggingface.co/modelcloud/"><img src="https://img.shields.io/badge/🤗%20Hugging%20Face-ModelCloud-%23ff8811.svg"></a>
|
|
33
|
+
</p>
|
|
34
|
+
|
|
35
|
+
## News
|
|
36
|
+
* 02/21/2025 [0.0.3](https://github.com/ModelCloud/Tokenicer/releases/tag/v0.0.3): Now `tokenicer` instance dynamically inherits the `native` `tokenizer.__class__` of tokenizer passed in or loaded via our `tokenicer.load()` api.
|
|
37
|
+
|
|
38
|
+
* 02/10/2025 [0.0.2](https://github.com/ModelCloud/Tokenicer/releases/tag/v0.0.2): 🤗 Initial release!
|
|
39
|
+
|
|
40
|
+
## Features:
|
|
41
|
+
|
|
42
|
+
* Compatible with all HF `Transformers` recognized tokenizers
|
|
43
|
+
* Auto-fix `models` not setting `padding_token`
|
|
44
|
+
* Auto-Fix `models` released with wrong `padding_token`: many `models` incorrectly use `eos_token` as `pad_token` which leads to subtle and hidden errors in post-training and inference when `batching` is used which is almost always.
|
|
45
|
+
* Zero external dependency outside of `Transformers`
|
|
46
|
+
|
|
47
|
+
## Upcoming Features:
|
|
48
|
+
|
|
49
|
+
* Add `automatic` tokenizer validation to `model` `training` and subsequent `inference` so that not only tokenizer config but actual `decode`/`encode` are 100% re-validated on model load. Often the case, `inference` and `training` engines modifies the traditional tokenizers causing subtle and inaccurate output when `inference` performed on a platform that is disjointed from the `trainer`.
|
|
50
|
+
|
|
51
|
+
## Install
|
|
52
|
+
|
|
53
|
+
### PIP/UV
|
|
54
|
+
|
|
55
|
+
```bash
|
|
56
|
+
pip install -v tokenicer
|
|
57
|
+
uv pip install -v tokenicer
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
### Install from source
|
|
61
|
+
|
|
62
|
+
```bash
|
|
63
|
+
# clone repo
|
|
64
|
+
git clone https://github.com/ModelCloud/Tokencier.git && cd Tokenicer
|
|
65
|
+
|
|
66
|
+
# compile
|
|
67
|
+
pip install -v .
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
## Usage
|
|
71
|
+
|
|
72
|
+
* Replace all calls to `AutoTokenizer.from_pretrained()` with `Tokenizer.load()`: args are 100% compatible with `AutoTokenizer`
|
|
73
|
+
|
|
74
|
+
```py
|
|
75
|
+
# Replace `AutoTokenizer.from_pretrained()`
|
|
76
|
+
# from tokenizer import AutoTokenizer
|
|
77
|
+
# tokenizer = AutoTokenizer.from_pretrained('Qwen/Qwen2.5-0.5B-Instruct')
|
|
78
|
+
|
|
79
|
+
# With `Tokenicer.load()`
|
|
80
|
+
from tokenicer import Tokenicer
|
|
81
|
+
|
|
82
|
+
# Returns `Tokenicer` instance that inherits original `Qwen2TokenizerFast` type.
|
|
83
|
+
tokenizer = Tokenicer.load('Qwen/Qwen2.5-0.5B-Instruct')
|
|
84
|
+
|
|
85
|
+
# That's it! Toke(n)icer has auto-fixed Qwen2.5-0.5B-Instruct's incorrect `pad_token`.
|
|
86
|
+
# Now this this model can be `trained` and `inferenced` correctly with `batch` and `masks`.
|
|
87
|
+
# Now use the new tokenizer like any normal HF PretrainedTokenizer(Fast)
|
|
88
|
+
print(f"pad_token: `{tokenizer.pad_token}`")
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
## Citation
|
|
92
|
+
|
|
93
|
+
```
|
|
94
|
+
@misc{gptqmodel,
|
|
95
|
+
author = {ModelCloud.ai and qubitium@modelcloud.ai},
|
|
96
|
+
title = {Toke(n)icer},
|
|
97
|
+
year = {2025},
|
|
98
|
+
publisher = {GitHub},
|
|
99
|
+
journal = {GitHub repository},
|
|
100
|
+
howpublished = {\url{https://github.com/modelcloud/tokenicer}},
|
|
101
|
+
note = {Contact: qubitium@modelcloud.ai}
|
|
102
|
+
}
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
<h1 align="center">Toke(n)icer</h1>
|
|
2
|
+
<p align="center">A (nicer) tokenizer you want to use for model inference and training: with all known peventable gotchas normalized or auto-fixed.</p>
|
|
3
|
+
<p align="center">
|
|
4
|
+
<a href="https://github.com/ModelCloud/Tokenicer/releases" style="text-decoration:none;"><img alt="GitHub release" src="https://img.shields.io/github/release/ModelCloud/Tokenicer.svg"></a>
|
|
5
|
+
<a href="https://pypi.org/project/tokenicer/" style="text-decoration:none;"><img alt="PyPI - Version" src="https://img.shields.io/pypi/v/tokenicer"></a>
|
|
6
|
+
<a href="https://pepy.tech/projects/tokenicer" style="text-decoration:none;"><img src="https://static.pepy.tech/badge/tokenicer" alt="PyPI Downloads"></a>
|
|
7
|
+
<a href="https://github.com/ModelCloud/tokenicer/blob/main/LICENSE"><img src="https://img.shields.io/pypi/l/tokenicer"></a>
|
|
8
|
+
<a href="https://huggingface.co/modelcloud/"><img src="https://img.shields.io/badge/🤗%20Hugging%20Face-ModelCloud-%23ff8811.svg"></a>
|
|
9
|
+
</p>
|
|
10
|
+
|
|
11
|
+
## News
|
|
12
|
+
* 02/21/2025 [0.0.3](https://github.com/ModelCloud/Tokenicer/releases/tag/v0.0.3): Now `tokenicer` instance dynamically inherits the `native` `tokenizer.__class__` of tokenizer passed in or loaded via our `tokenicer.load()` api.
|
|
13
|
+
|
|
14
|
+
* 02/10/2025 [0.0.2](https://github.com/ModelCloud/Tokenicer/releases/tag/v0.0.2): 🤗 Initial release!
|
|
15
|
+
|
|
16
|
+
## Features:
|
|
17
|
+
|
|
18
|
+
* Compatible with all HF `Transformers` recognized tokenizers
|
|
19
|
+
* Auto-fix `models` not setting `padding_token`
|
|
20
|
+
* Auto-Fix `models` released with wrong `padding_token`: many `models` incorrectly use `eos_token` as `pad_token` which leads to subtle and hidden errors in post-training and inference when `batching` is used which is almost always.
|
|
21
|
+
* Zero external dependency outside of `Transformers`
|
|
22
|
+
|
|
23
|
+
## Upcoming Features:
|
|
24
|
+
|
|
25
|
+
* Add `automatic` tokenizer validation to `model` `training` and subsequent `inference` so that not only tokenizer config but actual `decode`/`encode` are 100% re-validated on model load. Often the case, `inference` and `training` engines modifies the traditional tokenizers causing subtle and inaccurate output when `inference` performed on a platform that is disjointed from the `trainer`.
|
|
26
|
+
|
|
27
|
+
## Install
|
|
28
|
+
|
|
29
|
+
### PIP/UV
|
|
30
|
+
|
|
31
|
+
```bash
|
|
32
|
+
pip install -v tokenicer
|
|
33
|
+
uv pip install -v tokenicer
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
### Install from source
|
|
37
|
+
|
|
38
|
+
```bash
|
|
39
|
+
# clone repo
|
|
40
|
+
git clone https://github.com/ModelCloud/Tokencier.git && cd Tokenicer
|
|
41
|
+
|
|
42
|
+
# compile
|
|
43
|
+
pip install -v .
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
## Usage
|
|
47
|
+
|
|
48
|
+
* Replace all calls to `AutoTokenizer.from_pretrained()` with `Tokenizer.load()`: args are 100% compatible with `AutoTokenizer`
|
|
49
|
+
|
|
50
|
+
```py
|
|
51
|
+
# Replace `AutoTokenizer.from_pretrained()`
|
|
52
|
+
# from tokenizer import AutoTokenizer
|
|
53
|
+
# tokenizer = AutoTokenizer.from_pretrained('Qwen/Qwen2.5-0.5B-Instruct')
|
|
54
|
+
|
|
55
|
+
# With `Tokenicer.load()`
|
|
56
|
+
from tokenicer import Tokenicer
|
|
57
|
+
|
|
58
|
+
# Returns `Tokenicer` instance that inherits original `Qwen2TokenizerFast` type.
|
|
59
|
+
tokenizer = Tokenicer.load('Qwen/Qwen2.5-0.5B-Instruct')
|
|
60
|
+
|
|
61
|
+
# That's it! Toke(n)icer has auto-fixed Qwen2.5-0.5B-Instruct's incorrect `pad_token`.
|
|
62
|
+
# Now this this model can be `trained` and `inferenced` correctly with `batch` and `masks`.
|
|
63
|
+
# Now use the new tokenizer like any normal HF PretrainedTokenizer(Fast)
|
|
64
|
+
print(f"pad_token: `{tokenizer.pad_token}`")
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
## Citation
|
|
68
|
+
|
|
69
|
+
```
|
|
70
|
+
@misc{gptqmodel,
|
|
71
|
+
author = {ModelCloud.ai and qubitium@modelcloud.ai},
|
|
72
|
+
title = {Toke(n)icer},
|
|
73
|
+
year = {2025},
|
|
74
|
+
publisher = {GitHub},
|
|
75
|
+
journal = {GitHub repository},
|
|
76
|
+
howpublished = {\url{https://github.com/modelcloud/tokenicer}},
|
|
77
|
+
note = {Contact: qubitium@modelcloud.ai}
|
|
78
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
transformers
|
tokenicer-0.0.3/setup.py
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
# Copyright 2025 ModelCloud.ai
|
|
2
|
+
# Copyright 2025 qubitium@modelcloud.ai
|
|
3
|
+
# Contact: qubitium@modelcloud.ai, x.com/qubitium
|
|
4
|
+
#
|
|
5
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
6
|
+
# you may not use this file except in compliance with the License.
|
|
7
|
+
# You may obtain a copy of the License at
|
|
8
|
+
#
|
|
9
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
10
|
+
#
|
|
11
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
12
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
13
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
14
|
+
# See the License for the specific language governing permissions and
|
|
15
|
+
# limitations under the License.
|
|
16
|
+
|
|
17
|
+
from pathlib import Path
|
|
18
|
+
|
|
19
|
+
from setuptools import find_packages, setup
|
|
20
|
+
|
|
21
|
+
__version__ = "0.0.3"
|
|
22
|
+
|
|
23
|
+
with open("requirements.txt") as f:
|
|
24
|
+
requirements = f.read().splitlines()
|
|
25
|
+
|
|
26
|
+
setup(
|
|
27
|
+
name="tokenicer",
|
|
28
|
+
version=__version__,
|
|
29
|
+
author="ModelCloud",
|
|
30
|
+
author_email="qubitium@modelcloud.ai",
|
|
31
|
+
description="A (nicer) tokenizer you want to use for model `inference` and `training`: with all known peventable `gotchas` normalized or auto-fixed.",
|
|
32
|
+
long_description=(Path(__file__).parent / "README.md").read_text(encoding="UTF-8"),
|
|
33
|
+
long_description_content_type="text/markdown",
|
|
34
|
+
url="https://github.com/ModelCloud/Tokenicer",
|
|
35
|
+
packages=find_packages(),
|
|
36
|
+
install_requires=requirements,
|
|
37
|
+
platform=["linux", "windows", "darwin"],
|
|
38
|
+
classifiers=[
|
|
39
|
+
"Programming Language :: Python :: 3",
|
|
40
|
+
"License :: OSI Approved :: Apache Software License",
|
|
41
|
+
"Operating System :: OS Independent",
|
|
42
|
+
],
|
|
43
|
+
python_requires=">=3",
|
|
44
|
+
)
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
# Copyright 2025 ModelCloud.ai
|
|
2
|
+
# Copyright 2025 qubitium@modelcloud.ai
|
|
3
|
+
# Contact: qubitium@modelcloud.ai, x.com/qubitium
|
|
4
|
+
#
|
|
5
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
6
|
+
# you may not use this file except in compliance with the License.
|
|
7
|
+
# You may obtain a copy of the License at
|
|
8
|
+
#
|
|
9
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
10
|
+
#
|
|
11
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
12
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
13
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
14
|
+
# See the License for the specific language governing permissions and
|
|
15
|
+
# limitations under the License.
|
|
16
|
+
|
|
17
|
+
from .tokenicer import Tokenicer
|
|
18
|
+
|
|
19
|
+
__all__ = ["Tokenicer"]
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
# Copyright 2025 ModelCloud.ai
|
|
2
|
+
# Copyright 2025 qubitium@modelcloud.ai
|
|
3
|
+
# Contact: qubitium@modelcloud.ai, x.com/qubitium
|
|
4
|
+
#
|
|
5
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
6
|
+
# you may not use this file except in compliance with the License.
|
|
7
|
+
# You may obtain a copy of the License at
|
|
8
|
+
#
|
|
9
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
10
|
+
#
|
|
11
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
12
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
13
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
14
|
+
# See the License for the specific language governing permissions and
|
|
15
|
+
# limitations under the License.
|
|
16
|
+
|
|
17
|
+
from collections import namedtuple
|
|
18
|
+
|
|
19
|
+
DEFAULT_PAD_TOKENS = [
|
|
20
|
+
"<|finetune_right_pad_id|>",
|
|
21
|
+
"<|pad|>",
|
|
22
|
+
"<pad>",
|
|
23
|
+
"<|unk|>",
|
|
24
|
+
"<unk>"
|
|
25
|
+
]
|
|
26
|
+
|
|
27
|
+
TOKEN_TUPLE = namedtuple("TokenTuple", ["token", "token_id"])
|
|
28
|
+
|
|
29
|
+
MODEL_PAD_TOKEN_MAP = {
|
|
30
|
+
"llama": TOKEN_TUPLE(token='<|finetune_right_pad_id|>', token_id=128004),
|
|
31
|
+
"qwen2_5_vl": TOKEN_TUPLE(token='<|vision_pad|>', token_id=151654),
|
|
32
|
+
"qwen2_vl": TOKEN_TUPLE(token='<|vision_pad|>', token_id=151654),
|
|
33
|
+
"qwen2": TOKEN_TUPLE(token='<|fim_pad|>', token_id=151662),
|
|
34
|
+
"deepseek_v3": TOKEN_TUPLE(token='<|▁pad▁|>', token_id=2),
|
|
35
|
+
"mpt": TOKEN_TUPLE(token='<|padding|>', token_id=1)
|
|
36
|
+
}
|
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
# Copyright 2025 ModelCloud.ai
|
|
2
|
+
# Copyright 2025 qubitium@modelcloud.ai
|
|
3
|
+
# Contact: qubitium@modelcloud.ai, x.com/qubitium
|
|
4
|
+
#
|
|
5
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
6
|
+
# you may not use this file except in compliance with the License.
|
|
7
|
+
# You may obtain a copy of the License at
|
|
8
|
+
#
|
|
9
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
10
|
+
#
|
|
11
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
12
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
13
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
14
|
+
# See the License for the specific language governing permissions and
|
|
15
|
+
# limitations under the License.
|
|
16
|
+
|
|
17
|
+
import logging
|
|
18
|
+
from typing import List, Optional, Union
|
|
19
|
+
|
|
20
|
+
from transformers import AutoTokenizer, PreTrainedModel, PreTrainedTokenizerBase
|
|
21
|
+
|
|
22
|
+
from .const import DEFAULT_PAD_TOKENS, MODEL_PAD_TOKEN_MAP
|
|
23
|
+
from .util import auto_config, candidate_id, config_path
|
|
24
|
+
|
|
25
|
+
logger = logging.getLogger(__name__)
|
|
26
|
+
logging.basicConfig(level=logging.INFO)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class Tokenicer():
|
|
30
|
+
|
|
31
|
+
def __init__(self):
|
|
32
|
+
pass
|
|
33
|
+
|
|
34
|
+
@classmethod
|
|
35
|
+
def load(cls, pretrained_model_name_or_path: Union[str, PreTrainedTokenizerBase], strict: bool = False, pad_tokens: Optional[List[Union[str, int]]] = None, **kwargs):
|
|
36
|
+
if pretrained_model_name_or_path is None:
|
|
37
|
+
raise ValueError("Tokenicer: `pretrained_model_name_or_path` cannot be `None`.")
|
|
38
|
+
|
|
39
|
+
trust_remote_code = kwargs.get('trust_remote_code', False)
|
|
40
|
+
|
|
41
|
+
if isinstance(pretrained_model_name_or_path, PreTrainedTokenizerBase):
|
|
42
|
+
tokenizer = pretrained_model_name_or_path
|
|
43
|
+
path = config_path(tokenizer)
|
|
44
|
+
elif isinstance(pretrained_model_name_or_path, str):
|
|
45
|
+
tokenizer = AutoTokenizer.from_pretrained(pretrained_model_name_or_path, **kwargs)
|
|
46
|
+
if isinstance(tokenizer, PreTrainedTokenizerBase):
|
|
47
|
+
path = pretrained_model_name_or_path
|
|
48
|
+
else:
|
|
49
|
+
raise ValueError("Tokenicer: Failed to initialize `tokenizer`: please ensure the `pretrained_model_name_or_path` is set correctly.")
|
|
50
|
+
else:
|
|
51
|
+
raise ValueError(f"Tokenicer: Unsupported `pretrained_model_name_or_path` type: Expected `str` or `PreTrainedTokenizerBase`, actual = `{type(pretrained_model_name_or_path)}`.")
|
|
52
|
+
|
|
53
|
+
model_config = auto_config(path, trust_remote_code)
|
|
54
|
+
|
|
55
|
+
if model_config is None:
|
|
56
|
+
logger.warning(
|
|
57
|
+
"Tokenicer: Auto model config retrieval from `pretrained_model_name_or_path` failed. "
|
|
58
|
+
"Please pass a valid `model_or_path` argument to `auto_assign_pad_token()`.",
|
|
59
|
+
)
|
|
60
|
+
|
|
61
|
+
# dynamically change Tokenicer's type to tokenizer's
|
|
62
|
+
tokenizer_cls = type(tokenizer)
|
|
63
|
+
tokenicer_cls_wrapper = type(f"{tokenizer_cls.__name__}", (cls, tokenizer_cls), {})
|
|
64
|
+
|
|
65
|
+
t = tokenicer_cls_wrapper()
|
|
66
|
+
t.tokenizer = tokenizer
|
|
67
|
+
t.model_config = model_config
|
|
68
|
+
t.auto_fix_pad_token(strict=strict, pad_tokens=pad_tokens)
|
|
69
|
+
return t
|
|
70
|
+
|
|
71
|
+
def auto_fix_pad_token(
|
|
72
|
+
self,
|
|
73
|
+
model_or_path: Optional[Union[str, PreTrainedModel]] = None,
|
|
74
|
+
pad_tokens: Optional[List[Union[str, int]]] = None,
|
|
75
|
+
strict: bool = False,
|
|
76
|
+
):
|
|
77
|
+
if model_or_path is not None:
|
|
78
|
+
if isinstance(model_or_path, str):
|
|
79
|
+
model_config = auto_config(model_or_path, self.tokenizer.trust_remote_code)
|
|
80
|
+
elif isinstance(model_or_path, PreTrainedModel):
|
|
81
|
+
model_config = getattr(model_or_path, "config", None)
|
|
82
|
+
else:
|
|
83
|
+
raise ValueError(
|
|
84
|
+
f"Tokenicer: Unsupported `model_or_path` type: Expected `str` or `PreTrainedModel`, actual = `{type(model_or_path)}`.")
|
|
85
|
+
|
|
86
|
+
if model_config is None:
|
|
87
|
+
raise ValueError("Tokenicer: Can not retrieve config from the provided `model_or_path`.")
|
|
88
|
+
else:
|
|
89
|
+
if self.model_config is not None:
|
|
90
|
+
model_config = self.model_config
|
|
91
|
+
else:
|
|
92
|
+
raise ValueError(
|
|
93
|
+
"Tokenicer: Auto model config retrieval from `pretrained_model_name_or_path` failed. "
|
|
94
|
+
"Please pass a valid `model_or_path` argument to `auto_assign_pad_token()`.",
|
|
95
|
+
)
|
|
96
|
+
|
|
97
|
+
self.auto_fix_model_config(model_config)
|
|
98
|
+
|
|
99
|
+
pad_token_id = model_config.pad_token_id
|
|
100
|
+
|
|
101
|
+
if pad_token_id is None or pad_token_id in [model_config.bos_token_id, model_config.eos_token_id]:
|
|
102
|
+
pad_token_id = self._auto_map_pad_token(model_config=model_config, pad_tokens=pad_tokens)
|
|
103
|
+
|
|
104
|
+
if not strict:
|
|
105
|
+
if pad_token_id is None and self.tokenizer.eos_token_id is not None:
|
|
106
|
+
pad_token_id = self.tokenizer.eos_token_id
|
|
107
|
+
logger.warning(
|
|
108
|
+
"Tokenicer: Auto model config unable to fix `pad_token`, Use tokenizer.eos_token as pad_token"
|
|
109
|
+
"pad_token = eos_token, There may be problems with the model during training or inference."
|
|
110
|
+
"It is recommended that you manually pass a `pad_tokens` to `load()`",
|
|
111
|
+
)
|
|
112
|
+
|
|
113
|
+
if pad_token_id is None:
|
|
114
|
+
raise ValueError(
|
|
115
|
+
"Tokenicer: Model tokenizer requires fixing but we are unable to auto-fix `pad_token`. Please consult model docs and pass `pad_tokens` to `load()` or set `strict`= False."
|
|
116
|
+
)
|
|
117
|
+
|
|
118
|
+
self.tokenizer.pad_token_id = pad_token_id
|
|
119
|
+
self.tokenizer.pad_token = self.tokenizer.decode([pad_token_id])
|
|
120
|
+
|
|
121
|
+
logger.info(f"Tokenicer: Auto fixed pad_token_id={pad_token_id} (token='{self.tokenizer.pad_token}').")
|
|
122
|
+
|
|
123
|
+
def _auto_map_pad_token(self, model_config, pad_tokens) -> Optional[int]:
|
|
124
|
+
pad_token_id = None
|
|
125
|
+
|
|
126
|
+
vocab = self.tokenizer.get_vocab()
|
|
127
|
+
|
|
128
|
+
# Prioritize matching of pad token entered by the user
|
|
129
|
+
if pad_tokens is not None:
|
|
130
|
+
pad_token_id = candidate_id(pad_tokens, vocab)
|
|
131
|
+
|
|
132
|
+
# Match MODEL_PAD_TOKEN_MAP to get pad token
|
|
133
|
+
if pad_token_id is None and MODEL_PAD_TOKEN_MAP.get(model_config.model_type, None) is not None:
|
|
134
|
+
token_tuple = MODEL_PAD_TOKEN_MAP.get(model_config.model_type)
|
|
135
|
+
pad_token = token_tuple.token
|
|
136
|
+
token_id = vocab.get(pad_token, None)
|
|
137
|
+
if token_id is not None and token_id == token_tuple.token_id:
|
|
138
|
+
pad_token_id = token_id
|
|
139
|
+
|
|
140
|
+
# Match DEFAULT_PAD_TOKENS to get pad token
|
|
141
|
+
if pad_token_id is None:
|
|
142
|
+
pad_token_id = candidate_id(DEFAULT_PAD_TOKENS, vocab)
|
|
143
|
+
|
|
144
|
+
# Use eos_token as pad token
|
|
145
|
+
if pad_token_id is None:
|
|
146
|
+
if isinstance(model_config.eos_token_id, list) and model_config.eos_token_id:
|
|
147
|
+
pad_token_id = model_config.eos_token_id[0]
|
|
148
|
+
else:
|
|
149
|
+
pad_token_id = model_config.eos_token_id
|
|
150
|
+
|
|
151
|
+
return pad_token_id
|
|
152
|
+
|
|
153
|
+
def auto_fix_model_config(self, model_config):
|
|
154
|
+
if model_config.bos_token_id is None and self.tokenizer.bos_token_id is not None:
|
|
155
|
+
model_config.bos_token = self.tokenizer.bos_token
|
|
156
|
+
model_config.bos_token_id = self.tokenizer.bos_token_id
|
|
157
|
+
|
|
158
|
+
if model_config.eos_token_id is None and self.tokenizer.eos_token_id is not None:
|
|
159
|
+
model_config.eos_token = self.tokenizer.eos_token
|
|
160
|
+
model_config.eos_token_id = self.tokenizer.eos_token_id
|
|
161
|
+
|
|
162
|
+
def __getattribute__(self, name):
|
|
163
|
+
try:
|
|
164
|
+
return super().__getattribute__("tokenizer").__getattribute__(name)
|
|
165
|
+
except AttributeError:
|
|
166
|
+
return super().__getattribute__(name)
|
|
167
|
+
|
|
168
|
+
def __getattr__(self, name):
|
|
169
|
+
return getattr(self.tokenizer, name)
|
|
170
|
+
|
|
171
|
+
def __call__(self, data, **kwargs):
|
|
172
|
+
return self.tokenizer(data, **kwargs)
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
# Copyright 2025 ModelCloud.ai
|
|
2
|
+
# Copyright 2025 qubitium@modelcloud.ai
|
|
3
|
+
# Contact: qubitium@modelcloud.ai, x.com/qubitium
|
|
4
|
+
#
|
|
5
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
6
|
+
# you may not use this file except in compliance with the License.
|
|
7
|
+
# You may obtain a copy of the License at
|
|
8
|
+
#
|
|
9
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
10
|
+
#
|
|
11
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
12
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
13
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
14
|
+
# See the License for the specific language governing permissions and
|
|
15
|
+
# limitations under the License.
|
|
16
|
+
|
|
17
|
+
from typing import List, Optional, Union
|
|
18
|
+
|
|
19
|
+
from transformers import AutoConfig, PretrainedConfig
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def candidate_ids(token_list: List[Union[str, int]], vocab: dict) -> List[Optional[int]]:
|
|
23
|
+
token_ids = []
|
|
24
|
+
for item in token_list:
|
|
25
|
+
if isinstance(item, str):
|
|
26
|
+
val = vocab.get(item)
|
|
27
|
+
if val is not None:
|
|
28
|
+
token_ids.append(val)
|
|
29
|
+
elif isinstance(item, int):
|
|
30
|
+
if 0 <= item < len(vocab):
|
|
31
|
+
token_ids.append(item)
|
|
32
|
+
return token_ids
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def candidate_id(token_list: List[Union[str, int]], vocab: dict) -> Optional[int]:
|
|
36
|
+
token_ids = candidate_ids(token_list=token_list, vocab=vocab)
|
|
37
|
+
return token_ids[0] if token_ids else None
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def config_path(obj) -> Optional[str]:
|
|
41
|
+
path = getattr(obj, "name_or_path", None)
|
|
42
|
+
return path
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def auto_config(path, trust_remote) -> Optional[PretrainedConfig]:
|
|
46
|
+
config = AutoConfig.from_pretrained(path, trust_remote_code=trust_remote)
|
|
47
|
+
model_config = None
|
|
48
|
+
if isinstance(config, PretrainedConfig):
|
|
49
|
+
model_config = config
|
|
50
|
+
return model_config
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
Metadata-Version: 2.2
|
|
2
|
+
Name: tokenicer
|
|
3
|
+
Version: 0.0.3
|
|
4
|
+
Summary: A (nicer) tokenizer you want to use for model `inference` and `training`: with all known peventable `gotchas` normalized or auto-fixed.
|
|
5
|
+
Home-page: https://github.com/ModelCloud/Tokenicer
|
|
6
|
+
Author: ModelCloud
|
|
7
|
+
Author-email: qubitium@modelcloud.ai
|
|
8
|
+
Classifier: Programming Language :: Python :: 3
|
|
9
|
+
Classifier: License :: OSI Approved :: Apache Software License
|
|
10
|
+
Classifier: Operating System :: OS Independent
|
|
11
|
+
Requires-Python: >=3
|
|
12
|
+
Description-Content-Type: text/markdown
|
|
13
|
+
License-File: LICENSE
|
|
14
|
+
Requires-Dist: transformers
|
|
15
|
+
Dynamic: author
|
|
16
|
+
Dynamic: author-email
|
|
17
|
+
Dynamic: classifier
|
|
18
|
+
Dynamic: description
|
|
19
|
+
Dynamic: description-content-type
|
|
20
|
+
Dynamic: home-page
|
|
21
|
+
Dynamic: requires-dist
|
|
22
|
+
Dynamic: requires-python
|
|
23
|
+
Dynamic: summary
|
|
24
|
+
|
|
25
|
+
<h1 align="center">Toke(n)icer</h1>
|
|
26
|
+
<p align="center">A (nicer) tokenizer you want to use for model inference and training: with all known peventable gotchas normalized or auto-fixed.</p>
|
|
27
|
+
<p align="center">
|
|
28
|
+
<a href="https://github.com/ModelCloud/Tokenicer/releases" style="text-decoration:none;"><img alt="GitHub release" src="https://img.shields.io/github/release/ModelCloud/Tokenicer.svg"></a>
|
|
29
|
+
<a href="https://pypi.org/project/tokenicer/" style="text-decoration:none;"><img alt="PyPI - Version" src="https://img.shields.io/pypi/v/tokenicer"></a>
|
|
30
|
+
<a href="https://pepy.tech/projects/tokenicer" style="text-decoration:none;"><img src="https://static.pepy.tech/badge/tokenicer" alt="PyPI Downloads"></a>
|
|
31
|
+
<a href="https://github.com/ModelCloud/tokenicer/blob/main/LICENSE"><img src="https://img.shields.io/pypi/l/tokenicer"></a>
|
|
32
|
+
<a href="https://huggingface.co/modelcloud/"><img src="https://img.shields.io/badge/🤗%20Hugging%20Face-ModelCloud-%23ff8811.svg"></a>
|
|
33
|
+
</p>
|
|
34
|
+
|
|
35
|
+
## News
|
|
36
|
+
* 02/21/2025 [0.0.3](https://github.com/ModelCloud/Tokenicer/releases/tag/v0.0.3): Now `tokenicer` instance dynamically inherits the `native` `tokenizer.__class__` of tokenizer passed in or loaded via our `tokenicer.load()` api.
|
|
37
|
+
|
|
38
|
+
* 02/10/2025 [0.0.2](https://github.com/ModelCloud/Tokenicer/releases/tag/v0.0.2): 🤗 Initial release!
|
|
39
|
+
|
|
40
|
+
## Features:
|
|
41
|
+
|
|
42
|
+
* Compatible with all HF `Transformers` recognized tokenizers
|
|
43
|
+
* Auto-fix `models` not setting `padding_token`
|
|
44
|
+
* Auto-Fix `models` released with wrong `padding_token`: many `models` incorrectly use `eos_token` as `pad_token` which leads to subtle and hidden errors in post-training and inference when `batching` is used which is almost always.
|
|
45
|
+
* Zero external dependency outside of `Transformers`
|
|
46
|
+
|
|
47
|
+
## Upcoming Features:
|
|
48
|
+
|
|
49
|
+
* Add `automatic` tokenizer validation to `model` `training` and subsequent `inference` so that not only tokenizer config but actual `decode`/`encode` are 100% re-validated on model load. Often the case, `inference` and `training` engines modifies the traditional tokenizers causing subtle and inaccurate output when `inference` performed on a platform that is disjointed from the `trainer`.
|
|
50
|
+
|
|
51
|
+
## Install
|
|
52
|
+
|
|
53
|
+
### PIP/UV
|
|
54
|
+
|
|
55
|
+
```bash
|
|
56
|
+
pip install -v tokenicer
|
|
57
|
+
uv pip install -v tokenicer
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
### Install from source
|
|
61
|
+
|
|
62
|
+
```bash
|
|
63
|
+
# clone repo
|
|
64
|
+
git clone https://github.com/ModelCloud/Tokencier.git && cd Tokenicer
|
|
65
|
+
|
|
66
|
+
# compile
|
|
67
|
+
pip install -v .
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
## Usage
|
|
71
|
+
|
|
72
|
+
* Replace all calls to `AutoTokenizer.from_pretrained()` with `Tokenizer.load()`: args are 100% compatible with `AutoTokenizer`
|
|
73
|
+
|
|
74
|
+
```py
|
|
75
|
+
# Replace `AutoTokenizer.from_pretrained()`
|
|
76
|
+
# from tokenizer import AutoTokenizer
|
|
77
|
+
# tokenizer = AutoTokenizer.from_pretrained('Qwen/Qwen2.5-0.5B-Instruct')
|
|
78
|
+
|
|
79
|
+
# With `Tokenicer.load()`
|
|
80
|
+
from tokenicer import Tokenicer
|
|
81
|
+
|
|
82
|
+
# Returns `Tokenicer` instance that inherits original `Qwen2TokenizerFast` type.
|
|
83
|
+
tokenizer = Tokenicer.load('Qwen/Qwen2.5-0.5B-Instruct')
|
|
84
|
+
|
|
85
|
+
# That's it! Toke(n)icer has auto-fixed Qwen2.5-0.5B-Instruct's incorrect `pad_token`.
|
|
86
|
+
# Now this this model can be `trained` and `inferenced` correctly with `batch` and `masks`.
|
|
87
|
+
# Now use the new tokenizer like any normal HF PretrainedTokenizer(Fast)
|
|
88
|
+
print(f"pad_token: `{tokenizer.pad_token}`")
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
## Citation
|
|
92
|
+
|
|
93
|
+
```
|
|
94
|
+
@misc{gptqmodel,
|
|
95
|
+
author = {ModelCloud.ai and qubitium@modelcloud.ai},
|
|
96
|
+
title = {Toke(n)icer},
|
|
97
|
+
year = {2025},
|
|
98
|
+
publisher = {GitHub},
|
|
99
|
+
journal = {GitHub repository},
|
|
100
|
+
howpublished = {\url{https://github.com/modelcloud/tokenicer}},
|
|
101
|
+
note = {Contact: qubitium@modelcloud.ai}
|
|
102
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
LICENSE
|
|
2
|
+
MANIFEST.in
|
|
3
|
+
README.md
|
|
4
|
+
requirements.txt
|
|
5
|
+
setup.py
|
|
6
|
+
tokenicer/__init__.py
|
|
7
|
+
tokenicer/const.py
|
|
8
|
+
tokenicer/tokenicer.py
|
|
9
|
+
tokenicer/util.py
|
|
10
|
+
tokenicer.egg-info/PKG-INFO
|
|
11
|
+
tokenicer.egg-info/SOURCES.txt
|
|
12
|
+
tokenicer.egg-info/dependency_links.txt
|
|
13
|
+
tokenicer.egg-info/requires.txt
|
|
14
|
+
tokenicer.egg-info/top_level.txt
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
transformers
|
tokenicer-0.0.1.dev0/PKG-INFO
DELETED
|
@@ -1,23 +0,0 @@
|
|
|
1
|
-
Metadata-Version: 2.2
|
|
2
|
-
Name: tokenicer
|
|
3
|
-
Version: 0.0.1.dev0
|
|
4
|
-
Summary: A nicer tokenizer
|
|
5
|
-
Home-page: https://github.com/ModelCloud/Tokenicer
|
|
6
|
-
Author: ModelCloud
|
|
7
|
-
Author-email: qubitium@modelcloud.ai
|
|
8
|
-
Classifier: Programming Language :: Python :: 3
|
|
9
|
-
Classifier: License :: OSI Approved :: Apache Software License
|
|
10
|
-
Classifier: Operating System :: OS Independent
|
|
11
|
-
Requires-Python: >=3
|
|
12
|
-
Description-Content-Type: text/markdown
|
|
13
|
-
License-File: LICENSE
|
|
14
|
-
Dynamic: author
|
|
15
|
-
Dynamic: author-email
|
|
16
|
-
Dynamic: classifier
|
|
17
|
-
Dynamic: description
|
|
18
|
-
Dynamic: description-content-type
|
|
19
|
-
Dynamic: home-page
|
|
20
|
-
Dynamic: requires-python
|
|
21
|
-
Dynamic: summary
|
|
22
|
-
|
|
23
|
-
# Tokenicer
|
tokenicer-0.0.1.dev0/README.md
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
# Tokenicer
|
tokenicer-0.0.1.dev0/setup.py
DELETED
|
@@ -1,24 +0,0 @@
|
|
|
1
|
-
from setuptools import setup, find_packages
|
|
2
|
-
from pathlib import Path
|
|
3
|
-
|
|
4
|
-
__version__ = "0.0.1-dev"
|
|
5
|
-
|
|
6
|
-
setup(
|
|
7
|
-
name="tokenicer",
|
|
8
|
-
version=__version__,
|
|
9
|
-
author="ModelCloud",
|
|
10
|
-
author_email="qubitium@modelcloud.ai",
|
|
11
|
-
description="A nicer tokenizer",
|
|
12
|
-
long_description=(Path(__file__).parent / "README.md").read_text(encoding="UTF-8"),
|
|
13
|
-
long_description_content_type="text/markdown",
|
|
14
|
-
url="https://github.com/ModelCloud/Tokenicer",
|
|
15
|
-
packages=find_packages(),
|
|
16
|
-
install_requires=[],
|
|
17
|
-
platform=["linux"],
|
|
18
|
-
classifiers=[
|
|
19
|
-
"Programming Language :: Python :: 3",
|
|
20
|
-
"License :: OSI Approved :: Apache Software License",
|
|
21
|
-
"Operating System :: OS Independent",
|
|
22
|
-
],
|
|
23
|
-
python_requires=">=3",
|
|
24
|
-
)
|
|
File without changes
|
|
@@ -1,23 +0,0 @@
|
|
|
1
|
-
Metadata-Version: 2.2
|
|
2
|
-
Name: tokenicer
|
|
3
|
-
Version: 0.0.1.dev0
|
|
4
|
-
Summary: A nicer tokenizer
|
|
5
|
-
Home-page: https://github.com/ModelCloud/Tokenicer
|
|
6
|
-
Author: ModelCloud
|
|
7
|
-
Author-email: qubitium@modelcloud.ai
|
|
8
|
-
Classifier: Programming Language :: Python :: 3
|
|
9
|
-
Classifier: License :: OSI Approved :: Apache Software License
|
|
10
|
-
Classifier: Operating System :: OS Independent
|
|
11
|
-
Requires-Python: >=3
|
|
12
|
-
Description-Content-Type: text/markdown
|
|
13
|
-
License-File: LICENSE
|
|
14
|
-
Dynamic: author
|
|
15
|
-
Dynamic: author-email
|
|
16
|
-
Dynamic: classifier
|
|
17
|
-
Dynamic: description
|
|
18
|
-
Dynamic: description-content-type
|
|
19
|
-
Dynamic: home-page
|
|
20
|
-
Dynamic: requires-python
|
|
21
|
-
Dynamic: summary
|
|
22
|
-
|
|
23
|
-
# Tokenicer
|
|
File without changes
|
|
File without changes
|
|
File without changes
|