faTok 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fatok/__about__.py +7 -0
- fatok/__init__.py +18 -0
- fatok/cli.py +95 -0
- fatok/data/special_tokens_map.json +9 -0
- fatok/data/tokenizer.json +2075 -0
- fatok/data/tokenizer_config.json +55 -0
- fatok/data/training_stats.json +13 -0
- fatok/tokenizer.py +215 -0
- fatok-0.1.0.dist-info/METADATA +170 -0
- fatok-0.1.0.dist-info/RECORD +13 -0
- fatok-0.1.0.dist-info/WHEEL +4 -0
- fatok-0.1.0.dist-info/entry_points.txt +2 -0
- fatok-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
{
|
|
2
|
+
"added_tokens_decoder": {
|
|
3
|
+
"1": {
|
|
4
|
+
"content": "<s>",
|
|
5
|
+
"lstrip": false,
|
|
6
|
+
"normalized": false,
|
|
7
|
+
"rstrip": false,
|
|
8
|
+
"single_word": false,
|
|
9
|
+
"special": true
|
|
10
|
+
},
|
|
11
|
+
"2": {
|
|
12
|
+
"content": "</s>",
|
|
13
|
+
"lstrip": false,
|
|
14
|
+
"normalized": false,
|
|
15
|
+
"rstrip": false,
|
|
16
|
+
"single_word": false,
|
|
17
|
+
"special": true
|
|
18
|
+
},
|
|
19
|
+
"3": {
|
|
20
|
+
"content": "<pad>",
|
|
21
|
+
"lstrip": false,
|
|
22
|
+
"normalized": false,
|
|
23
|
+
"rstrip": false,
|
|
24
|
+
"single_word": false,
|
|
25
|
+
"special": true
|
|
26
|
+
},
|
|
27
|
+
"4": {
|
|
28
|
+
"content": "### سوال:",
|
|
29
|
+
"lstrip": false,
|
|
30
|
+
"normalized": false,
|
|
31
|
+
"rstrip": false,
|
|
32
|
+
"single_word": false,
|
|
33
|
+
"special": true
|
|
34
|
+
},
|
|
35
|
+
"5": {
|
|
36
|
+
"content": "### پاسخ:",
|
|
37
|
+
"lstrip": false,
|
|
38
|
+
"normalized": false,
|
|
39
|
+
"rstrip": false,
|
|
40
|
+
"single_word": false,
|
|
41
|
+
"special": true
|
|
42
|
+
}
|
|
43
|
+
},
|
|
44
|
+
"additional_special_tokens": [
|
|
45
|
+
"### سوال:",
|
|
46
|
+
"### پاسخ:"
|
|
47
|
+
],
|
|
48
|
+
"bos_token": "<s>",
|
|
49
|
+
"clean_up_tokenization_spaces": false,
|
|
50
|
+
"eos_token": "</s>",
|
|
51
|
+
"model_max_length": 2048,
|
|
52
|
+
"pad_token": "<pad>",
|
|
53
|
+
"tokenizer_class": "PreTrainedTokenizerFast",
|
|
54
|
+
"unk_token": null
|
|
55
|
+
}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
{
|
|
2
|
+
"vocab_size": 598,
|
|
3
|
+
"special_tokens": [
|
|
4
|
+
"",
|
|
5
|
+
"<s>",
|
|
6
|
+
"</s>",
|
|
7
|
+
"<pad>",
|
|
8
|
+
"### سوال:",
|
|
9
|
+
"### پاسخ:"
|
|
10
|
+
],
|
|
11
|
+
"train_lines": 544,
|
|
12
|
+
"note": "Default small (4K) Persian BPE tokenizer bundled with faTok. Replace data/tokenizer.json with your own 32K tokenizer to upgrade."
|
|
13
|
+
}
|
fatok/tokenizer.py
ADDED
|
@@ -0,0 +1,215 @@
|
|
|
1
|
+
"""The :class:`FaTok` wrapper around a HuggingFace ``PreTrainedTokenizerFast``.
|
|
2
|
+
|
|
3
|
+
The tokenizer JSON ships inside the package (``fatok/data/tokenizer.json``).
|
|
4
|
+
It is a byte-level BPE tokenizer with Persian-aware normalization:
|
|
5
|
+
|
|
6
|
+
* NFKC
|
|
7
|
+
* Arabic Yeh / Keheh normalization (ي → ی, ك → ک, ى → ی)
|
|
8
|
+
* Stripping of Arabic diacritics (tashkeel) and Tatweel
|
|
9
|
+
* ZWNJ (نیمفاصله) preserved as part of words
|
|
10
|
+
* GPT-2 style pre-tokenization adapted for Persian + ZWNJ
|
|
11
|
+
|
|
12
|
+
The bundled tokenizer is small (4K vocab) so the package is functional
|
|
13
|
+
out of the box. To upgrade, drop your own ``tokenizer.json`` into
|
|
14
|
+
``site-packages/fatok/data/`` and reload — no code changes needed.
|
|
15
|
+
"""
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import json
|
|
19
|
+
import os
|
|
20
|
+
from functools import lru_cache
|
|
21
|
+
from pathlib import Path
|
|
22
|
+
from typing import Any, Iterable, List, Sequence, Union
|
|
23
|
+
|
|
24
|
+
# Resolve the bundled data directory (works in wheel, sdist, and editable installs).
|
|
25
|
+
_DATA_DIR = Path(__file__).resolve().parent / "data"
|
|
26
|
+
_DEFAULT_TOKENIZER_PATH = _DATA_DIR / "tokenizer.json"
|
|
27
|
+
|
|
28
|
+
TextLike = Union[str, Sequence[str]]
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class FaTok:
|
|
32
|
+
"""A Persian BPE tokenizer.
|
|
33
|
+
|
|
34
|
+
Parameters
|
|
35
|
+
----------
|
|
36
|
+
path:
|
|
37
|
+
Optional path to a custom ``tokenizer.json``. When omitted, the
|
|
38
|
+
bundled Persian tokenizer is loaded.
|
|
39
|
+
|
|
40
|
+
Examples
|
|
41
|
+
--------
|
|
42
|
+
>>> from fatok import FaTok
|
|
43
|
+
>>> tok = FaTok()
|
|
44
|
+
>>> ids = tok.encode("سلام دنیا")
|
|
45
|
+
>>> tok.decode(ids)
|
|
46
|
+
'سلام دنیا'
|
|
47
|
+
"""
|
|
48
|
+
|
|
49
|
+
def __init__(self, path: Union[str, os.PathLike, None] = None):
|
|
50
|
+
if path is None:
|
|
51
|
+
path = _DEFAULT_TOKENIZER_PATH
|
|
52
|
+
path = Path(path)
|
|
53
|
+
if not path.exists():
|
|
54
|
+
raise FileNotFoundError(
|
|
55
|
+
f"Tokenizer file not found: {path}. "
|
|
56
|
+
f"Either pass `path=` to point at a tokenizer.json, "
|
|
57
|
+
f"or reinstall faTok to restore the bundled one."
|
|
58
|
+
)
|
|
59
|
+
# Import locally so that ``import fatok`` itself stays cheap if the
|
|
60
|
+
# user just wants the version / module metadata.
|
|
61
|
+
from tokenizers import Tokenizer
|
|
62
|
+
|
|
63
|
+
self._tok = Tokenizer.from_file(str(path))
|
|
64
|
+
self._path = path
|
|
65
|
+
|
|
66
|
+
# ------------------------------------------------------------------
|
|
67
|
+
# Core encode / decode
|
|
68
|
+
# ------------------------------------------------------------------
|
|
69
|
+
def encode(self, text: str, add_special_tokens: bool = False) -> List[int]:
|
|
70
|
+
"""Encode a single string into a list of token ids.
|
|
71
|
+
|
|
72
|
+
Parameters
|
|
73
|
+
----------
|
|
74
|
+
text:
|
|
75
|
+
Input string. Will be normalized (Yeh/Kaf, diacritics) inside the tokenizer.
|
|
76
|
+
add_special_tokens:
|
|
77
|
+
If True, prepend BOS (``<s>``) and append EOS (``</s>``).
|
|
78
|
+
This is implemented in the wrapper (rather than via the
|
|
79
|
+
tokenizer's post-processor) so it works regardless of how
|
|
80
|
+
the bundled tokenizer.json was configured.
|
|
81
|
+
"""
|
|
82
|
+
enc = self._tok.encode(text, add_special_tokens=False)
|
|
83
|
+
ids = list(enc.ids)
|
|
84
|
+
if add_special_tokens:
|
|
85
|
+
bos = self.token_to_id("<s>")
|
|
86
|
+
eos = self.token_to_id("</s>")
|
|
87
|
+
if bos != -1:
|
|
88
|
+
ids.insert(0, bos)
|
|
89
|
+
if eos != -1:
|
|
90
|
+
ids.append(eos)
|
|
91
|
+
return ids
|
|
92
|
+
|
|
93
|
+
def encode_batch(
|
|
94
|
+
self, texts: Iterable[str], add_special_tokens: bool = False
|
|
95
|
+
) -> List[List[int]]:
|
|
96
|
+
"""Encode a batch of strings."""
|
|
97
|
+
out = [
|
|
98
|
+
e.ids
|
|
99
|
+
for e in self._tok.encode_batch(list(texts), add_special_tokens=False)
|
|
100
|
+
]
|
|
101
|
+
if add_special_tokens:
|
|
102
|
+
bos = self.token_to_id("<s>")
|
|
103
|
+
eos = self.token_to_id("</s>")
|
|
104
|
+
for row in out:
|
|
105
|
+
if bos != -1:
|
|
106
|
+
row.insert(0, bos)
|
|
107
|
+
if eos != -1:
|
|
108
|
+
row.append(eos)
|
|
109
|
+
return out
|
|
110
|
+
|
|
111
|
+
def decode(self, ids: Sequence[int], skip_special_tokens: bool = True) -> str:
|
|
112
|
+
"""Decode token ids back into a string."""
|
|
113
|
+
return self._tok.decode(list(ids), skip_special_tokens=skip_special_tokens)
|
|
114
|
+
|
|
115
|
+
def decode_batch(
|
|
116
|
+
self, batch_ids: Sequence[Sequence[int]], skip_special_tokens: bool = True
|
|
117
|
+
) -> List[str]:
|
|
118
|
+
"""Decode a batch of id-lists."""
|
|
119
|
+
return self._tok.decode_batch(
|
|
120
|
+
[list(ids) for ids in batch_ids],
|
|
121
|
+
skip_special_tokens=skip_special_tokens,
|
|
122
|
+
)
|
|
123
|
+
|
|
124
|
+
# ------------------------------------------------------------------
|
|
125
|
+
# Convenience
|
|
126
|
+
# ------------------------------------------------------------------
|
|
127
|
+
@property
|
|
128
|
+
def vocab_size(self) -> int:
|
|
129
|
+
"""Total vocab size including special tokens."""
|
|
130
|
+
return self._tok.get_vocab_size()
|
|
131
|
+
|
|
132
|
+
@property
|
|
133
|
+
def vocab(self) -> dict:
|
|
134
|
+
"""Token-string → id map."""
|
|
135
|
+
return self._tok.get_vocab(with_added_tokens=True)
|
|
136
|
+
|
|
137
|
+
def token_to_id(self, token: str) -> int:
|
|
138
|
+
"""Look up a token string; returns -1 if absent.
|
|
139
|
+
|
|
140
|
+
(The underlying ``tokenizers`` library returns ``None`` for missing
|
|
141
|
+
tokens; we normalize to ``-1`` to match the GPT-2 / HuggingFace
|
|
142
|
+
convention so callers can ``if id >= 0:``.)
|
|
143
|
+
"""
|
|
144
|
+
out = self._tok.token_to_id(token)
|
|
145
|
+
return -1 if out is None else out
|
|
146
|
+
|
|
147
|
+
def id_to_token(self, idx: int) -> Union[str, None]:
|
|
148
|
+
"""Look up an id; returns None if out of range."""
|
|
149
|
+
return self._tok.id_to_token(idx)
|
|
150
|
+
|
|
151
|
+
def normalize(self, text: str) -> str:
|
|
152
|
+
"""Apply the tokenizer's Persian normalization to a string without tokenizing."""
|
|
153
|
+
return self._tok.normalizer.normalize_str(text)
|
|
154
|
+
|
|
155
|
+
def count(self, text: str, add_special_tokens: bool = False) -> int:
|
|
156
|
+
"""Number of tokens after encoding ``text``. Cheap shortcut for budget checks."""
|
|
157
|
+
return len(self.encode(text, add_special_tokens=add_special_tokens))
|
|
158
|
+
|
|
159
|
+
# ------------------------------------------------------------------
|
|
160
|
+
# HuggingFace interop
|
|
161
|
+
# ------------------------------------------------------------------
|
|
162
|
+
def to_hf(self, **kwargs: Any):
|
|
163
|
+
"""Return a ``transformers.PreTrainedTokenizerFast`` wrapping the same tokenizer.
|
|
164
|
+
|
|
165
|
+
Any keyword arguments override the saved config (e.g. ``model_max_length=4096``).
|
|
166
|
+
"""
|
|
167
|
+
from transformers import PreTrainedTokenizerFast
|
|
168
|
+
|
|
169
|
+
defaults = dict(
|
|
170
|
+
bos_token="<s>",
|
|
171
|
+
eos_token="</s>",
|
|
172
|
+
pad_token="<pad>",
|
|
173
|
+
unk_token=None,
|
|
174
|
+
additional_special_tokens=["### سوال:", "### پاسخ:"],
|
|
175
|
+
model_max_length=2048,
|
|
176
|
+
)
|
|
177
|
+
defaults.update(kwargs)
|
|
178
|
+
return PreTrainedTokenizerFast(tokenizer_object=self._tok, **defaults)
|
|
179
|
+
|
|
180
|
+
# ------------------------------------------------------------------
|
|
181
|
+
# Internals
|
|
182
|
+
# ------------------------------------------------------------------
|
|
183
|
+
def __repr__(self) -> str:
|
|
184
|
+
return f"FaTok(vocab_size={self.vocab_size}, path={self._path.name!s})"
|
|
185
|
+
|
|
186
|
+
def __len__(self) -> int:
|
|
187
|
+
return self.vocab_size
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
# ---------------------------------------------------------------------------
|
|
191
|
+
# Module-level lazy accessor
|
|
192
|
+
# ---------------------------------------------------------------------------
|
|
193
|
+
@lru_cache(maxsize=1)
|
|
194
|
+
def get_default() -> FaTok:
|
|
195
|
+
"""Return a process-wide cached default tokenizer.
|
|
196
|
+
|
|
197
|
+
Useful when you call encode/decode in a hot loop and don't want to
|
|
198
|
+
pay the file-load cost on every call.
|
|
199
|
+
"""
|
|
200
|
+
return FaTok()
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def info() -> dict:
|
|
204
|
+
"""Return metadata about the bundled tokenizer (vocab size, special tokens, path)."""
|
|
205
|
+
stats_path = _DATA_DIR / "training_stats.json"
|
|
206
|
+
if stats_path.exists():
|
|
207
|
+
with open(stats_path, encoding="utf-8") as f:
|
|
208
|
+
stats = json.load(f)
|
|
209
|
+
else:
|
|
210
|
+
stats = {}
|
|
211
|
+
return {
|
|
212
|
+
"path": str(_DEFAULT_TOKENIZER_PATH),
|
|
213
|
+
"vocab_size": FaTok().vocab_size,
|
|
214
|
+
**stats,
|
|
215
|
+
}
|
|
@@ -0,0 +1,170 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: faTok
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: A Persian-first BPE tokenizer with built-in Persian normalization (Yeh/Kaf, diacritics, ZWNJ-aware).
|
|
5
|
+
Project-URL: Homepage, https://github.com/your-org/faTok
|
|
6
|
+
Project-URL: Documentation, https://github.com/your-org/faTok#readme
|
|
7
|
+
Project-URL: Issues, https://github.com/your-org/faTok/issues
|
|
8
|
+
Project-URL: Source, https://github.com/your-org/faTok
|
|
9
|
+
Author: faTok contributors
|
|
10
|
+
License: MIT License
|
|
11
|
+
|
|
12
|
+
Copyright (c) 2026 faTok contributors
|
|
13
|
+
|
|
14
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
15
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
16
|
+
in the Software without restriction, including without limitation the rights
|
|
17
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
18
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
19
|
+
furnished to do so, subject to the following conditions:
|
|
20
|
+
|
|
21
|
+
The above copyright notice and this permission notice shall be included in all
|
|
22
|
+
copies or substantial portions of the Software.
|
|
23
|
+
|
|
24
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
25
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
26
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
27
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
28
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
29
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
30
|
+
SOFTWARE.
|
|
31
|
+
License-File: LICENSE
|
|
32
|
+
Keywords: bpe,byte-level,farsi,huggingface,nlp,persian,subword,tokenizer
|
|
33
|
+
Classifier: Development Status :: 4 - Beta
|
|
34
|
+
Classifier: Intended Audience :: Developers
|
|
35
|
+
Classifier: Intended Audience :: Science/Research
|
|
36
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
37
|
+
Classifier: Natural Language :: Persian
|
|
38
|
+
Classifier: Operating System :: OS Independent
|
|
39
|
+
Classifier: Programming Language :: Python :: 3
|
|
40
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
41
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
42
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
43
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
44
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
45
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
46
|
+
Classifier: Topic :: Text Processing :: Linguistic
|
|
47
|
+
Requires-Python: >=3.9
|
|
48
|
+
Requires-Dist: pyarrow<19,>=15
|
|
49
|
+
Requires-Dist: tokenizers<0.21,>=0.20
|
|
50
|
+
Requires-Dist: transformers<5,>=4.46
|
|
51
|
+
Provides-Extra: dev
|
|
52
|
+
Requires-Dist: build>=1.0; extra == 'dev'
|
|
53
|
+
Requires-Dist: pytest>=7.0; extra == 'dev'
|
|
54
|
+
Requires-Dist: twine>=4.0; extra == 'dev'
|
|
55
|
+
Description-Content-Type: text/markdown
|
|
56
|
+
|
|
57
|
+
# faTok
|
|
58
|
+
|
|
59
|
+
**faTok** — *Persian-first BPE tokenizer.*
|
|
60
|
+
|
|
61
|
+
A small, dependency-light Python package that ships a byte-level BPE
|
|
62
|
+
tokenizer with built-in Persian normalization:
|
|
63
|
+
|
|
64
|
+
- Arabic Yeh / Keheh normalization (`ي → ی`, `ك → ک`, `ى → ی`)
|
|
65
|
+
- Stripping of Arabic diacritics (`tashkeel`) and Tatweel (`ـ`)
|
|
66
|
+
- ZWNJ (نیمفاصله) preserved as part of words
|
|
67
|
+
- GPT-2 style pre-tokenization adapted for Persian + ZWNJ
|
|
68
|
+
|
|
69
|
+
The bundled tokenizer is small (≈ 4K vocab) so the package is usable
|
|
70
|
+
immediately. Drop your own `tokenizer.json` into
|
|
71
|
+
`site-packages/fatok/data/` to upgrade to a larger vocab — no code
|
|
72
|
+
changes required.
|
|
73
|
+
|
|
74
|
+
## Installation
|
|
75
|
+
|
|
76
|
+
```bash
|
|
77
|
+
pip install fatok
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
For local development:
|
|
81
|
+
|
|
82
|
+
```bash
|
|
83
|
+
git clone https://github.com/your-org/faTok
|
|
84
|
+
cd faTok
|
|
85
|
+
pip install -e .
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
## Quick start
|
|
89
|
+
|
|
90
|
+
```python
|
|
91
|
+
from fatok import FaTok
|
|
92
|
+
|
|
93
|
+
tok = FaTok()
|
|
94
|
+
ids = tok.encode("سلام دنیا، حالت چطوره؟")
|
|
95
|
+
print(ids) # [123, 456, 789, ...]
|
|
96
|
+
print(tok.decode(ids)) # سلام دنیا، حالت چطوره؟
|
|
97
|
+
print(tok.vocab_size) # 4096 (bundled) or whatever you replaced it with
|
|
98
|
+
print(tok.count("یک متن")) # number of tokens
|
|
99
|
+
print(tok.normalize("سَلام كِيف حالُك")) # سلام کیف حالک
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
### Batch encode / decode
|
|
103
|
+
|
|
104
|
+
```python
|
|
105
|
+
ids_batch = tok.encode_batch(["جملهی اول", "جملهی دوم"])
|
|
106
|
+
texts = tok.decode_batch(ids_batch)
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
### Add special tokens
|
|
110
|
+
|
|
111
|
+
```python
|
|
112
|
+
ids = tok.encode("### سوال: پایتخت ایران؟", add_special_tokens=True)
|
|
113
|
+
# ids starts with <s> and ends with </s>
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
### Use the underlying HuggingFace tokenizer
|
|
117
|
+
|
|
118
|
+
```python
|
|
119
|
+
hf = tok.to_hf(model_max_length=4096)
|
|
120
|
+
# hf is a transformers.PreTrainedTokenizerFast — pass to any HF model
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
## Command-line interface
|
|
124
|
+
|
|
125
|
+
```bash
|
|
126
|
+
fatok info
|
|
127
|
+
fatok encode "سلام دنیا"
|
|
128
|
+
fatok encode "سلام دنیا" --json
|
|
129
|
+
fatok decode 123 456 789
|
|
130
|
+
fatok count "یک متن نسبتاً طولانی برای شمارش توکنها"
|
|
131
|
+
fatok normalize "سَلام كَيْف حالُك"
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
## Using your own tokenizer
|
|
135
|
+
|
|
136
|
+
After training your own Persian BPE tokenizer (for example with the
|
|
137
|
+
`train_persian_tokenizer_v2.ipynb` notebook), replace the bundled one:
|
|
138
|
+
|
|
139
|
+
```bash
|
|
140
|
+
cp /path/to/your/tokenizer.json "$(python -c 'import fatok, os; print(os.path.dirname(fatok.__file__))')/data/tokenizer.json"
|
|
141
|
+
```
|
|
142
|
+
|
|
143
|
+
Then re-import `fatok` and the new vocab is live.
|
|
144
|
+
|
|
145
|
+
## API reference
|
|
146
|
+
|
|
147
|
+
| Method | Returns | Description |
|
|
148
|
+
|---|---|---|
|
|
149
|
+
| `FaTok(path=None)` | `FaTok` | Load the bundled (or custom) tokenizer. |
|
|
150
|
+
| `tok.encode(text, add_special_tokens=False)` | `list[int]` | Encode a string. |
|
|
151
|
+
| `tok.encode_batch(texts, ...)` | `list[list[int]]` | Encode many strings. |
|
|
152
|
+
| `tok.decode(ids, skip_special_tokens=True)` | `str` | Decode ids. |
|
|
153
|
+
| `tok.decode_batch(batch_ids, ...)` | `list[str]` | Decode many. |
|
|
154
|
+
| `tok.normalize(text)` | `str` | Apply Persian normalization only. |
|
|
155
|
+
| `tok.count(text)` | `int` | Token count (cheap). |
|
|
156
|
+
| `tok.token_to_id(tok_str)` | `int` | `-1` if absent. |
|
|
157
|
+
| `tok.id_to_token(idx)` | `str \| None` | Reverse lookup. |
|
|
158
|
+
| `tok.vocab_size` | `int` | Including specials. |
|
|
159
|
+
| `tok.vocab` | `dict` | Token → id map. |
|
|
160
|
+
| `tok.to_hf(**kw)` | `PreTrainedTokenizerFast` | For HF ecosystem. |
|
|
161
|
+
|
|
162
|
+
## Versioning
|
|
163
|
+
|
|
164
|
+
This package follows the bundled tokenizer's vocab size: any change to
|
|
165
|
+
the bundled `tokenizer.json` (different vocab, different special tokens)
|
|
166
|
+
requires a minor-version bump.
|
|
167
|
+
|
|
168
|
+
## License
|
|
169
|
+
|
|
170
|
+
MIT — see [LICENSE](LICENSE).
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
fatok/__about__.py,sha256=yY1rqFv7FPcMDWUGaIOeH0rDQGqHcZgFd_97e5Wz2i8,241
|
|
2
|
+
fatok/__init__.py,sha256=rtdmUoS5wf7l5OPkqFPmM9QK2J6RxRjKG0QQjSSHolw,474
|
|
3
|
+
fatok/cli.py,sha256=uNaHaQh7PC7CegHR_mq6Y7NlvMTiDotIUenZHXny_wM,3009
|
|
4
|
+
fatok/tokenizer.py,sha256=FeUwSOlofH3ATEAa49AnHvnvN__4nzRT8Jta-Fkt8GI,7861
|
|
5
|
+
fatok/data/special_tokens_map.json,sha256=gDD19ozZic9dh0X1-Eugt3VSc6-RjF6hnoz3Y7V5o8Y,151
|
|
6
|
+
fatok/data/tokenizer.json,sha256=1SKhFuwl7Q3iH62EESCta3W3_kVNscUqY0cS-sWRbMk,34493
|
|
7
|
+
fatok/data/tokenizer_config.json,sha256=D6WE2qW2_L8QIDAulH-Z8Qz0Ws3kKELVqoWBjL2gYYY,1169
|
|
8
|
+
fatok/data/training_stats.json,sha256=5Ep-TLuJcmru0LkyNZ1m6CB-a4pFaxSi-2YrIrgXxLA,299
|
|
9
|
+
fatok-0.1.0.dist-info/METADATA,sha256=Ow_7r-Oj6q4l5FyKYgMjkWv7VuspQdlSoKRMHlnU8-s,6270
|
|
10
|
+
fatok-0.1.0.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
|
|
11
|
+
fatok-0.1.0.dist-info/entry_points.txt,sha256=eAUj82uSa99g31p7CvRXn5No4n2xmP9Q4nHb9th3jKQ,41
|
|
12
|
+
fatok-0.1.0.dist-info/licenses/LICENSE,sha256=BwFgikNxgXi5R4waukXsziu53qbxJbUU9QcvQs9B97U,1075
|
|
13
|
+
fatok-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 faTok contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|