faTok 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fatok-0.1.0/.gitignore +2 -0
- fatok-0.1.0/LICENSE +21 -0
- fatok-0.1.0/PKG-INFO +170 -0
- fatok-0.1.0/README.md +114 -0
- fatok-0.1.0/pyproject.toml +69 -0
- fatok-0.1.0/src/fatok/__about__.py +7 -0
- fatok-0.1.0/src/fatok/__init__.py +18 -0
- fatok-0.1.0/src/fatok/cli.py +95 -0
- fatok-0.1.0/src/fatok/data/special_tokens_map.json +9 -0
- fatok-0.1.0/src/fatok/data/tokenizer.json +2075 -0
- fatok-0.1.0/src/fatok/data/tokenizer_config.json +55 -0
- fatok-0.1.0/src/fatok/data/training_stats.json +13 -0
- fatok-0.1.0/src/fatok/tokenizer.py +215 -0
- fatok-0.1.0/tests/test_fatok.py +259 -0
fatok-0.1.0/.gitignore
ADDED
fatok-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 faTok contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
fatok-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,170 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: faTok
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: A Persian-first BPE tokenizer with built-in Persian normalization (Yeh/Kaf, diacritics, ZWNJ-aware).
|
|
5
|
+
Project-URL: Homepage, https://github.com/your-org/faTok
|
|
6
|
+
Project-URL: Documentation, https://github.com/your-org/faTok#readme
|
|
7
|
+
Project-URL: Issues, https://github.com/your-org/faTok/issues
|
|
8
|
+
Project-URL: Source, https://github.com/your-org/faTok
|
|
9
|
+
Author: faTok contributors
|
|
10
|
+
License: MIT License
|
|
11
|
+
|
|
12
|
+
Copyright (c) 2026 faTok contributors
|
|
13
|
+
|
|
14
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
15
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
16
|
+
in the Software without restriction, including without limitation the rights
|
|
17
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
18
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
19
|
+
furnished to do so, subject to the following conditions:
|
|
20
|
+
|
|
21
|
+
The above copyright notice and this permission notice shall be included in all
|
|
22
|
+
copies or substantial portions of the Software.
|
|
23
|
+
|
|
24
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
25
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
26
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
27
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
28
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
29
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
30
|
+
SOFTWARE.
|
|
31
|
+
License-File: LICENSE
|
|
32
|
+
Keywords: bpe,byte-level,farsi,huggingface,nlp,persian,subword,tokenizer
|
|
33
|
+
Classifier: Development Status :: 4 - Beta
|
|
34
|
+
Classifier: Intended Audience :: Developers
|
|
35
|
+
Classifier: Intended Audience :: Science/Research
|
|
36
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
37
|
+
Classifier: Natural Language :: Persian
|
|
38
|
+
Classifier: Operating System :: OS Independent
|
|
39
|
+
Classifier: Programming Language :: Python :: 3
|
|
40
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
41
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
42
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
43
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
44
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
45
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
46
|
+
Classifier: Topic :: Text Processing :: Linguistic
|
|
47
|
+
Requires-Python: >=3.9
|
|
48
|
+
Requires-Dist: pyarrow<19,>=15
|
|
49
|
+
Requires-Dist: tokenizers<0.21,>=0.20
|
|
50
|
+
Requires-Dist: transformers<5,>=4.46
|
|
51
|
+
Provides-Extra: dev
|
|
52
|
+
Requires-Dist: build>=1.0; extra == 'dev'
|
|
53
|
+
Requires-Dist: pytest>=7.0; extra == 'dev'
|
|
54
|
+
Requires-Dist: twine>=4.0; extra == 'dev'
|
|
55
|
+
Description-Content-Type: text/markdown
|
|
56
|
+
|
|
57
|
+
# faTok
|
|
58
|
+
|
|
59
|
+
**faTok** — *Persian-first BPE tokenizer.*
|
|
60
|
+
|
|
61
|
+
A small, dependency-light Python package that ships a byte-level BPE
|
|
62
|
+
tokenizer with built-in Persian normalization:
|
|
63
|
+
|
|
64
|
+
- Arabic Yeh / Keheh normalization (`ي → ی`, `ك → ک`, `ى → ی`)
|
|
65
|
+
- Stripping of Arabic diacritics (`tashkeel`) and Tatweel (`ـ`)
|
|
66
|
+
- ZWNJ (نیمفاصله) preserved as part of words
|
|
67
|
+
- GPT-2 style pre-tokenization adapted for Persian + ZWNJ
|
|
68
|
+
|
|
69
|
+
The bundled tokenizer is small (≈ 4K vocab) so the package is usable
|
|
70
|
+
immediately. Drop your own `tokenizer.json` into
|
|
71
|
+
`site-packages/fatok/data/` to upgrade to a larger vocab — no code
|
|
72
|
+
changes required.
|
|
73
|
+
|
|
74
|
+
## Installation
|
|
75
|
+
|
|
76
|
+
```bash
|
|
77
|
+
pip install fatok
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
For local development:
|
|
81
|
+
|
|
82
|
+
```bash
|
|
83
|
+
git clone https://github.com/your-org/faTok
|
|
84
|
+
cd faTok
|
|
85
|
+
pip install -e .
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
## Quick start
|
|
89
|
+
|
|
90
|
+
```python
|
|
91
|
+
from fatok import FaTok
|
|
92
|
+
|
|
93
|
+
tok = FaTok()
|
|
94
|
+
ids = tok.encode("سلام دنیا، حالت چطوره؟")
|
|
95
|
+
print(ids) # [123, 456, 789, ...]
|
|
96
|
+
print(tok.decode(ids)) # سلام دنیا، حالت چطوره؟
|
|
97
|
+
print(tok.vocab_size) # 4096 (bundled) or whatever you replaced it with
|
|
98
|
+
print(tok.count("یک متن")) # number of tokens
|
|
99
|
+
print(tok.normalize("سَلام كِيف حالُك")) # سلام کیف حالک
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
### Batch encode / decode
|
|
103
|
+
|
|
104
|
+
```python
|
|
105
|
+
ids_batch = tok.encode_batch(["جملهی اول", "جملهی دوم"])
|
|
106
|
+
texts = tok.decode_batch(ids_batch)
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
### Add special tokens
|
|
110
|
+
|
|
111
|
+
```python
|
|
112
|
+
ids = tok.encode("### سوال: پایتخت ایران؟", add_special_tokens=True)
|
|
113
|
+
# ids starts with <s> and ends with </s>
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
### Use the underlying HuggingFace tokenizer
|
|
117
|
+
|
|
118
|
+
```python
|
|
119
|
+
hf = tok.to_hf(model_max_length=4096)
|
|
120
|
+
# hf is a transformers.PreTrainedTokenizerFast — pass to any HF model
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
## Command-line interface
|
|
124
|
+
|
|
125
|
+
```bash
|
|
126
|
+
fatok info
|
|
127
|
+
fatok encode "سلام دنیا"
|
|
128
|
+
fatok encode "سلام دنیا" --json
|
|
129
|
+
fatok decode 123 456 789
|
|
130
|
+
fatok count "یک متن نسبتاً طولانی برای شمارش توکنها"
|
|
131
|
+
fatok normalize "سَلام كَيْف حالُك"
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
## Using your own tokenizer
|
|
135
|
+
|
|
136
|
+
After training your own Persian BPE tokenizer (for example with the
|
|
137
|
+
`train_persian_tokenizer_v2.ipynb` notebook), replace the bundled one:
|
|
138
|
+
|
|
139
|
+
```bash
|
|
140
|
+
cp /path/to/your/tokenizer.json "$(python -c 'import fatok, os; print(os.path.dirname(fatok.__file__))')/data/tokenizer.json"
|
|
141
|
+
```
|
|
142
|
+
|
|
143
|
+
Then re-import `fatok` and the new vocab is live.
|
|
144
|
+
|
|
145
|
+
## API reference
|
|
146
|
+
|
|
147
|
+
| Method | Returns | Description |
|
|
148
|
+
|---|---|---|
|
|
149
|
+
| `FaTok(path=None)` | `FaTok` | Load the bundled (or custom) tokenizer. |
|
|
150
|
+
| `tok.encode(text, add_special_tokens=False)` | `list[int]` | Encode a string. |
|
|
151
|
+
| `tok.encode_batch(texts, ...)` | `list[list[int]]` | Encode many strings. |
|
|
152
|
+
| `tok.decode(ids, skip_special_tokens=True)` | `str` | Decode ids. |
|
|
153
|
+
| `tok.decode_batch(batch_ids, ...)` | `list[str]` | Decode many. |
|
|
154
|
+
| `tok.normalize(text)` | `str` | Apply Persian normalization only. |
|
|
155
|
+
| `tok.count(text)` | `int` | Token count (cheap). |
|
|
156
|
+
| `tok.token_to_id(tok_str)` | `int` | `-1` if absent. |
|
|
157
|
+
| `tok.id_to_token(idx)` | `str \| None` | Reverse lookup. |
|
|
158
|
+
| `tok.vocab_size` | `int` | Including specials. |
|
|
159
|
+
| `tok.vocab` | `dict` | Token → id map. |
|
|
160
|
+
| `tok.to_hf(**kw)` | `PreTrainedTokenizerFast` | For HF ecosystem. |
|
|
161
|
+
|
|
162
|
+
## Versioning
|
|
163
|
+
|
|
164
|
+
This package follows the bundled tokenizer's vocab size: any change to
|
|
165
|
+
the bundled `tokenizer.json` (different vocab, different special tokens)
|
|
166
|
+
requires a minor-version bump.
|
|
167
|
+
|
|
168
|
+
## License
|
|
169
|
+
|
|
170
|
+
MIT — see [LICENSE](LICENSE).
|
fatok-0.1.0/README.md
ADDED
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
# faTok
|
|
2
|
+
|
|
3
|
+
**faTok** — *Persian-first BPE tokenizer.*
|
|
4
|
+
|
|
5
|
+
A small, dependency-light Python package that ships a byte-level BPE
|
|
6
|
+
tokenizer with built-in Persian normalization:
|
|
7
|
+
|
|
8
|
+
- Arabic Yeh / Keheh normalization (`ي → ی`, `ك → ک`, `ى → ی`)
|
|
9
|
+
- Stripping of Arabic diacritics (`tashkeel`) and Tatweel (`ـ`)
|
|
10
|
+
- ZWNJ (نیمفاصله) preserved as part of words
|
|
11
|
+
- GPT-2 style pre-tokenization adapted for Persian + ZWNJ
|
|
12
|
+
|
|
13
|
+
The bundled tokenizer is small (≈ 4K vocab) so the package is usable
|
|
14
|
+
immediately. Drop your own `tokenizer.json` into
|
|
15
|
+
`site-packages/fatok/data/` to upgrade to a larger vocab — no code
|
|
16
|
+
changes required.
|
|
17
|
+
|
|
18
|
+
## Installation
|
|
19
|
+
|
|
20
|
+
```bash
|
|
21
|
+
pip install fatok
|
|
22
|
+
```
|
|
23
|
+
|
|
24
|
+
For local development:
|
|
25
|
+
|
|
26
|
+
```bash
|
|
27
|
+
git clone https://github.com/your-org/faTok
|
|
28
|
+
cd faTok
|
|
29
|
+
pip install -e .
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
## Quick start
|
|
33
|
+
|
|
34
|
+
```python
|
|
35
|
+
from fatok import FaTok
|
|
36
|
+
|
|
37
|
+
tok = FaTok()
|
|
38
|
+
ids = tok.encode("سلام دنیا، حالت چطوره؟")
|
|
39
|
+
print(ids) # [123, 456, 789, ...]
|
|
40
|
+
print(tok.decode(ids)) # سلام دنیا، حالت چطوره؟
|
|
41
|
+
print(tok.vocab_size) # 4096 (bundled) or whatever you replaced it with
|
|
42
|
+
print(tok.count("یک متن")) # number of tokens
|
|
43
|
+
print(tok.normalize("سَلام كِيف حالُك")) # سلام کیف حالک
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
### Batch encode / decode
|
|
47
|
+
|
|
48
|
+
```python
|
|
49
|
+
ids_batch = tok.encode_batch(["جملهی اول", "جملهی دوم"])
|
|
50
|
+
texts = tok.decode_batch(ids_batch)
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
### Add special tokens
|
|
54
|
+
|
|
55
|
+
```python
|
|
56
|
+
ids = tok.encode("### سوال: پایتخت ایران؟", add_special_tokens=True)
|
|
57
|
+
# ids starts with <s> and ends with </s>
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
### Use the underlying HuggingFace tokenizer
|
|
61
|
+
|
|
62
|
+
```python
|
|
63
|
+
hf = tok.to_hf(model_max_length=4096)
|
|
64
|
+
# hf is a transformers.PreTrainedTokenizerFast — pass to any HF model
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
## Command-line interface
|
|
68
|
+
|
|
69
|
+
```bash
|
|
70
|
+
fatok info
|
|
71
|
+
fatok encode "سلام دنیا"
|
|
72
|
+
fatok encode "سلام دنیا" --json
|
|
73
|
+
fatok decode 123 456 789
|
|
74
|
+
fatok count "یک متن نسبتاً طولانی برای شمارش توکنها"
|
|
75
|
+
fatok normalize "سَلام كَيْف حالُك"
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
## Using your own tokenizer
|
|
79
|
+
|
|
80
|
+
After training your own Persian BPE tokenizer (for example with the
|
|
81
|
+
`train_persian_tokenizer_v2.ipynb` notebook), replace the bundled one:
|
|
82
|
+
|
|
83
|
+
```bash
|
|
84
|
+
cp /path/to/your/tokenizer.json "$(python -c 'import fatok, os; print(os.path.dirname(fatok.__file__))')/data/tokenizer.json"
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
Then re-import `fatok` and the new vocab is live.
|
|
88
|
+
|
|
89
|
+
## API reference
|
|
90
|
+
|
|
91
|
+
| Method | Returns | Description |
|
|
92
|
+
|---|---|---|
|
|
93
|
+
| `FaTok(path=None)` | `FaTok` | Load the bundled (or custom) tokenizer. |
|
|
94
|
+
| `tok.encode(text, add_special_tokens=False)` | `list[int]` | Encode a string. |
|
|
95
|
+
| `tok.encode_batch(texts, ...)` | `list[list[int]]` | Encode many strings. |
|
|
96
|
+
| `tok.decode(ids, skip_special_tokens=True)` | `str` | Decode ids. |
|
|
97
|
+
| `tok.decode_batch(batch_ids, ...)` | `list[str]` | Decode many. |
|
|
98
|
+
| `tok.normalize(text)` | `str` | Apply Persian normalization only. |
|
|
99
|
+
| `tok.count(text)` | `int` | Token count (cheap). |
|
|
100
|
+
| `tok.token_to_id(tok_str)` | `int` | `-1` if absent. |
|
|
101
|
+
| `tok.id_to_token(idx)` | `str \| None` | Reverse lookup. |
|
|
102
|
+
| `tok.vocab_size` | `int` | Including specials. |
|
|
103
|
+
| `tok.vocab` | `dict` | Token → id map. |
|
|
104
|
+
| `tok.to_hf(**kw)` | `PreTrainedTokenizerFast` | For HF ecosystem. |
|
|
105
|
+
|
|
106
|
+
## Versioning
|
|
107
|
+
|
|
108
|
+
This package follows the bundled tokenizer's vocab size: any change to
|
|
109
|
+
the bundled `tokenizer.json` (different vocab, different special tokens)
|
|
110
|
+
requires a minor-version bump.
|
|
111
|
+
|
|
112
|
+
## License
|
|
113
|
+
|
|
114
|
+
MIT — see [LICENSE](LICENSE).
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling>=1.21"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "faTok"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "A Persian-first BPE tokenizer with built-in Persian normalization (Yeh/Kaf, diacritics, ZWNJ-aware)."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.9"
|
|
11
|
+
license = { file = "LICENSE" }
|
|
12
|
+
authors = [{ name = "faTok contributors" }]
|
|
13
|
+
keywords = [
|
|
14
|
+
"persian", "farsi", "tokenizer", "bpe", "byte-level",
|
|
15
|
+
"nlp", "huggingface", "subword",
|
|
16
|
+
]
|
|
17
|
+
classifiers = [
|
|
18
|
+
"Development Status :: 4 - Beta",
|
|
19
|
+
"Intended Audience :: Developers",
|
|
20
|
+
"Intended Audience :: Science/Research",
|
|
21
|
+
"License :: OSI Approved :: MIT License",
|
|
22
|
+
"Natural Language :: Persian",
|
|
23
|
+
"Operating System :: OS Independent",
|
|
24
|
+
"Programming Language :: Python :: 3",
|
|
25
|
+
"Programming Language :: Python :: 3 :: Only",
|
|
26
|
+
"Programming Language :: Python :: 3.9",
|
|
27
|
+
"Programming Language :: Python :: 3.10",
|
|
28
|
+
"Programming Language :: Python :: 3.11",
|
|
29
|
+
"Programming Language :: Python :: 3.12",
|
|
30
|
+
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
|
31
|
+
"Topic :: Text Processing :: Linguistic",
|
|
32
|
+
]
|
|
33
|
+
dependencies = [
|
|
34
|
+
# Same pinned HF stack verified in requirements.txt — these are the
|
|
35
|
+
# versions known to import cleanly together with numpy 1.x and 2.x.
|
|
36
|
+
"tokenizers>=0.20,<0.21",
|
|
37
|
+
"transformers>=4.46,<5",
|
|
38
|
+
# pyarrow<19 is required when numpy<2 is in the user's environment;
|
|
39
|
+
# pyarrow 18.x supports BOTH numpy 1.x and 2.x so it's safe to pin here.
|
|
40
|
+
"pyarrow>=15,<19",
|
|
41
|
+
]
|
|
42
|
+
|
|
43
|
+
[project.optional-dependencies]
|
|
44
|
+
dev = [
|
|
45
|
+
"pytest>=7.0",
|
|
46
|
+
"build>=1.0",
|
|
47
|
+
"twine>=4.0",
|
|
48
|
+
]
|
|
49
|
+
|
|
50
|
+
[project.urls]
|
|
51
|
+
Homepage = "https://github.com/your-org/faTok"
|
|
52
|
+
Documentation = "https://github.com/your-org/faTok#readme"
|
|
53
|
+
Issues = "https://github.com/your-org/faTok/issues"
|
|
54
|
+
Source = "https://github.com/your-org/faTok"
|
|
55
|
+
|
|
56
|
+
[project.scripts]
|
|
57
|
+
fatok = "fatok.cli:main"
|
|
58
|
+
|
|
59
|
+
[tool.hatch.build.targets.wheel]
|
|
60
|
+
packages = ["src/fatok"]
|
|
61
|
+
|
|
62
|
+
[tool.hatch.build.targets.sdist]
|
|
63
|
+
include = [
|
|
64
|
+
"src/fatok",
|
|
65
|
+
"tests",
|
|
66
|
+
"README.md",
|
|
67
|
+
"LICENSE",
|
|
68
|
+
"pyproject.toml",
|
|
69
|
+
]
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
"""faTok — a Persian-first BPE tokenizer.
|
|
2
|
+
|
|
3
|
+
Public API
|
|
4
|
+
----------
|
|
5
|
+
from fatok import FaTok
|
|
6
|
+
|
|
7
|
+
tok = FaTok() # load the bundled Persian tokenizer
|
|
8
|
+
ids = tok.encode("سلام دنیا")
|
|
9
|
+
text = tok.decode(ids)
|
|
10
|
+
n = tok.vocab_size
|
|
11
|
+
hf = tok.to_hf() # underlying PreTrainedTokenizerFast
|
|
12
|
+
|
|
13
|
+
See ``FaTok`` for the full API.
|
|
14
|
+
"""
|
|
15
|
+
from .tokenizer import FaTok
|
|
16
|
+
from .__about__ import __version__
|
|
17
|
+
|
|
18
|
+
__all__ = ["FaTok", "__version__"]
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
"""``fatok`` command-line interface.
|
|
2
|
+
|
|
3
|
+
Examples
|
|
4
|
+
--------
|
|
5
|
+
fatok info
|
|
6
|
+
fatok encode "سلام دنیا"
|
|
7
|
+
fatok encode "سلام دنیا" --json
|
|
8
|
+
fatok decode 1234 5678 910
|
|
9
|
+
fatok count "یک متن نسبتا طولانی برای شمارش توکنها"
|
|
10
|
+
fatok normalize "سَلام كَيْف حالُك"
|
|
11
|
+
"""
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import argparse
|
|
15
|
+
import json
|
|
16
|
+
import sys
|
|
17
|
+
from typing import List, Optional
|
|
18
|
+
|
|
19
|
+
from .tokenizer import FaTok, info
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _build_parser() -> argparse.ArgumentParser:
|
|
23
|
+
p = argparse.ArgumentParser(
|
|
24
|
+
prog="fatok",
|
|
25
|
+
description="faTok — Persian BPE tokenizer.",
|
|
26
|
+
)
|
|
27
|
+
sub = p.add_subparsers(dest="cmd", required=True)
|
|
28
|
+
|
|
29
|
+
sub.add_parser("info", help="Print metadata about the bundled tokenizer.")
|
|
30
|
+
|
|
31
|
+
enc = sub.add_parser("encode", help="Encode text into token ids.")
|
|
32
|
+
enc.add_argument("text", help="Text to encode.")
|
|
33
|
+
enc.add_argument("--special-tokens", action="store_true",
|
|
34
|
+
help="Wrap with <s> ... </s>.")
|
|
35
|
+
enc.add_argument("--json", action="store_true",
|
|
36
|
+
help="Output JSON: {ids, n, text}.")
|
|
37
|
+
enc.add_argument("--path", help="Custom tokenizer.json path.")
|
|
38
|
+
|
|
39
|
+
dec = sub.add_parser("decode", help="Decode token ids back into text.")
|
|
40
|
+
dec.add_argument("ids", nargs="+", type=int, help="Token ids to decode.")
|
|
41
|
+
dec.add_argument("--keep-special-tokens", action="store_true",
|
|
42
|
+
help="Do not skip special tokens in the output.")
|
|
43
|
+
dec.add_argument("--path", help="Custom tokenizer.json path.")
|
|
44
|
+
|
|
45
|
+
cnt = sub.add_parser("count", help="Count tokens for a text.")
|
|
46
|
+
cnt.add_argument("text", help="Text to count tokens for.")
|
|
47
|
+
cnt.add_argument("--path", help="Custom tokenizer.json path.")
|
|
48
|
+
|
|
49
|
+
norm = sub.add_parser("normalize", help="Apply Persian normalization only.")
|
|
50
|
+
norm.add_argument("text", help="Text to normalize.")
|
|
51
|
+
norm.add_argument("--path", help="Custom tokenizer.json path.")
|
|
52
|
+
|
|
53
|
+
return p
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def main(argv: Optional[List[str]] = None) -> int:
|
|
57
|
+
args = _build_parser().parse_args(argv)
|
|
58
|
+
|
|
59
|
+
if args.cmd == "info":
|
|
60
|
+
data = info()
|
|
61
|
+
print(json.dumps(data, ensure_ascii=False, indent=2))
|
|
62
|
+
return 0
|
|
63
|
+
|
|
64
|
+
tok = FaTok(path=getattr(args, "path", None))
|
|
65
|
+
|
|
66
|
+
if args.cmd == "encode":
|
|
67
|
+
ids = tok.encode(args.text, add_special_tokens=args.special_tokens)
|
|
68
|
+
if args.json:
|
|
69
|
+
print(json.dumps({
|
|
70
|
+
"ids": ids,
|
|
71
|
+
"n": len(ids),
|
|
72
|
+
"text": args.text,
|
|
73
|
+
}, ensure_ascii=False))
|
|
74
|
+
else:
|
|
75
|
+
print(" ".join(str(i) for i in ids))
|
|
76
|
+
return 0
|
|
77
|
+
|
|
78
|
+
if args.cmd == "decode":
|
|
79
|
+
text = tok.decode(args.ids, skip_special_tokens=not args.keep_special_tokens)
|
|
80
|
+
print(text)
|
|
81
|
+
return 0
|
|
82
|
+
|
|
83
|
+
if args.cmd == "count":
|
|
84
|
+
print(tok.count(args.text))
|
|
85
|
+
return 0
|
|
86
|
+
|
|
87
|
+
if args.cmd == "normalize":
|
|
88
|
+
print(tok.normalize(args.text))
|
|
89
|
+
return 0
|
|
90
|
+
|
|
91
|
+
return 1
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
if __name__ == "__main__":
|
|
95
|
+
sys.exit(main())
|