faTok 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,55 @@
1
+ {
2
+ "added_tokens_decoder": {
3
+ "1": {
4
+ "content": "<s>",
5
+ "lstrip": false,
6
+ "normalized": false,
7
+ "rstrip": false,
8
+ "single_word": false,
9
+ "special": true
10
+ },
11
+ "2": {
12
+ "content": "</s>",
13
+ "lstrip": false,
14
+ "normalized": false,
15
+ "rstrip": false,
16
+ "single_word": false,
17
+ "special": true
18
+ },
19
+ "3": {
20
+ "content": "<pad>",
21
+ "lstrip": false,
22
+ "normalized": false,
23
+ "rstrip": false,
24
+ "single_word": false,
25
+ "special": true
26
+ },
27
+ "4": {
28
+ "content": "### سوال:",
29
+ "lstrip": false,
30
+ "normalized": false,
31
+ "rstrip": false,
32
+ "single_word": false,
33
+ "special": true
34
+ },
35
+ "5": {
36
+ "content": "### پاسخ:",
37
+ "lstrip": false,
38
+ "normalized": false,
39
+ "rstrip": false,
40
+ "single_word": false,
41
+ "special": true
42
+ }
43
+ },
44
+ "additional_special_tokens": [
45
+ "### سوال:",
46
+ "### پاسخ:"
47
+ ],
48
+ "bos_token": "<s>",
49
+ "clean_up_tokenization_spaces": false,
50
+ "eos_token": "</s>",
51
+ "model_max_length": 2048,
52
+ "pad_token": "<pad>",
53
+ "tokenizer_class": "PreTrainedTokenizerFast",
54
+ "unk_token": null
55
+ }
@@ -0,0 +1,13 @@
1
+ {
2
+ "vocab_size": 598,
3
+ "special_tokens": [
4
+ "",
5
+ "<s>",
6
+ "</s>",
7
+ "<pad>",
8
+ "### سوال:",
9
+ "### پاسخ:"
10
+ ],
11
+ "train_lines": 544,
12
+ "note": "Default small (4K) Persian BPE tokenizer bundled with faTok. Replace data/tokenizer.json with your own 32K tokenizer to upgrade."
13
+ }
fatok/tokenizer.py ADDED
@@ -0,0 +1,215 @@
1
+ """The :class:`FaTok` wrapper around a HuggingFace ``PreTrainedTokenizerFast``.
2
+
3
+ The tokenizer JSON ships inside the package (``fatok/data/tokenizer.json``).
4
+ It is a byte-level BPE tokenizer with Persian-aware normalization:
5
+
6
+ * NFKC
7
+ * Arabic Yeh / Keheh normalization (ي → ی, ك → ک, ى → ی)
8
+ * Stripping of Arabic diacritics (tashkeel) and Tatweel
9
+ * ZWNJ (نیم‌فاصله) preserved as part of words
10
+ * GPT-2 style pre-tokenization adapted for Persian + ZWNJ
11
+
12
+ The bundled tokenizer is small (4K vocab) so the package is functional
13
+ out of the box. To upgrade, drop your own ``tokenizer.json`` into
14
+ ``site-packages/fatok/data/`` and reload — no code changes needed.
15
+ """
16
+ from __future__ import annotations
17
+
18
+ import json
19
+ import os
20
+ from functools import lru_cache
21
+ from pathlib import Path
22
+ from typing import Any, Iterable, List, Sequence, Union
23
+
24
+ # Resolve the bundled data directory (works in wheel, sdist, and editable installs).
25
+ _DATA_DIR = Path(__file__).resolve().parent / "data"
26
+ _DEFAULT_TOKENIZER_PATH = _DATA_DIR / "tokenizer.json"
27
+
28
+ TextLike = Union[str, Sequence[str]]
29
+
30
+
31
+ class FaTok:
32
+ """A Persian BPE tokenizer.
33
+
34
+ Parameters
35
+ ----------
36
+ path:
37
+ Optional path to a custom ``tokenizer.json``. When omitted, the
38
+ bundled Persian tokenizer is loaded.
39
+
40
+ Examples
41
+ --------
42
+ >>> from fatok import FaTok
43
+ >>> tok = FaTok()
44
+ >>> ids = tok.encode("سلام دنیا")
45
+ >>> tok.decode(ids)
46
+ 'سلام دنیا'
47
+ """
48
+
49
+ def __init__(self, path: Union[str, os.PathLike, None] = None):
50
+ if path is None:
51
+ path = _DEFAULT_TOKENIZER_PATH
52
+ path = Path(path)
53
+ if not path.exists():
54
+ raise FileNotFoundError(
55
+ f"Tokenizer file not found: {path}. "
56
+ f"Either pass `path=` to point at a tokenizer.json, "
57
+ f"or reinstall faTok to restore the bundled one."
58
+ )
59
+ # Import locally so that ``import fatok`` itself stays cheap if the
60
+ # user just wants the version / module metadata.
61
+ from tokenizers import Tokenizer
62
+
63
+ self._tok = Tokenizer.from_file(str(path))
64
+ self._path = path
65
+
66
+ # ------------------------------------------------------------------
67
+ # Core encode / decode
68
+ # ------------------------------------------------------------------
69
+ def encode(self, text: str, add_special_tokens: bool = False) -> List[int]:
70
+ """Encode a single string into a list of token ids.
71
+
72
+ Parameters
73
+ ----------
74
+ text:
75
+ Input string. Will be normalized (Yeh/Kaf, diacritics) inside the tokenizer.
76
+ add_special_tokens:
77
+ If True, prepend BOS (``<s>``) and append EOS (``</s>``).
78
+ This is implemented in the wrapper (rather than via the
79
+ tokenizer's post-processor) so it works regardless of how
80
+ the bundled tokenizer.json was configured.
81
+ """
82
+ enc = self._tok.encode(text, add_special_tokens=False)
83
+ ids = list(enc.ids)
84
+ if add_special_tokens:
85
+ bos = self.token_to_id("<s>")
86
+ eos = self.token_to_id("</s>")
87
+ if bos != -1:
88
+ ids.insert(0, bos)
89
+ if eos != -1:
90
+ ids.append(eos)
91
+ return ids
92
+
93
+ def encode_batch(
94
+ self, texts: Iterable[str], add_special_tokens: bool = False
95
+ ) -> List[List[int]]:
96
+ """Encode a batch of strings."""
97
+ out = [
98
+ e.ids
99
+ for e in self._tok.encode_batch(list(texts), add_special_tokens=False)
100
+ ]
101
+ if add_special_tokens:
102
+ bos = self.token_to_id("<s>")
103
+ eos = self.token_to_id("</s>")
104
+ for row in out:
105
+ if bos != -1:
106
+ row.insert(0, bos)
107
+ if eos != -1:
108
+ row.append(eos)
109
+ return out
110
+
111
+ def decode(self, ids: Sequence[int], skip_special_tokens: bool = True) -> str:
112
+ """Decode token ids back into a string."""
113
+ return self._tok.decode(list(ids), skip_special_tokens=skip_special_tokens)
114
+
115
+ def decode_batch(
116
+ self, batch_ids: Sequence[Sequence[int]], skip_special_tokens: bool = True
117
+ ) -> List[str]:
118
+ """Decode a batch of id-lists."""
119
+ return self._tok.decode_batch(
120
+ [list(ids) for ids in batch_ids],
121
+ skip_special_tokens=skip_special_tokens,
122
+ )
123
+
124
+ # ------------------------------------------------------------------
125
+ # Convenience
126
+ # ------------------------------------------------------------------
127
+ @property
128
+ def vocab_size(self) -> int:
129
+ """Total vocab size including special tokens."""
130
+ return self._tok.get_vocab_size()
131
+
132
+ @property
133
+ def vocab(self) -> dict:
134
+ """Token-string → id map."""
135
+ return self._tok.get_vocab(with_added_tokens=True)
136
+
137
+ def token_to_id(self, token: str) -> int:
138
+ """Look up a token string; returns -1 if absent.
139
+
140
+ (The underlying ``tokenizers`` library returns ``None`` for missing
141
+ tokens; we normalize to ``-1`` to match the GPT-2 / HuggingFace
142
+ convention so callers can ``if id >= 0:``.)
143
+ """
144
+ out = self._tok.token_to_id(token)
145
+ return -1 if out is None else out
146
+
147
+ def id_to_token(self, idx: int) -> Union[str, None]:
148
+ """Look up an id; returns None if out of range."""
149
+ return self._tok.id_to_token(idx)
150
+
151
+ def normalize(self, text: str) -> str:
152
+ """Apply the tokenizer's Persian normalization to a string without tokenizing."""
153
+ return self._tok.normalizer.normalize_str(text)
154
+
155
+ def count(self, text: str, add_special_tokens: bool = False) -> int:
156
+ """Number of tokens after encoding ``text``. Cheap shortcut for budget checks."""
157
+ return len(self.encode(text, add_special_tokens=add_special_tokens))
158
+
159
+ # ------------------------------------------------------------------
160
+ # HuggingFace interop
161
+ # ------------------------------------------------------------------
162
+ def to_hf(self, **kwargs: Any):
163
+ """Return a ``transformers.PreTrainedTokenizerFast`` wrapping the same tokenizer.
164
+
165
+ Any keyword arguments override the saved config (e.g. ``model_max_length=4096``).
166
+ """
167
+ from transformers import PreTrainedTokenizerFast
168
+
169
+ defaults = dict(
170
+ bos_token="<s>",
171
+ eos_token="</s>",
172
+ pad_token="<pad>",
173
+ unk_token=None,
174
+ additional_special_tokens=["### سوال:", "### پاسخ:"],
175
+ model_max_length=2048,
176
+ )
177
+ defaults.update(kwargs)
178
+ return PreTrainedTokenizerFast(tokenizer_object=self._tok, **defaults)
179
+
180
+ # ------------------------------------------------------------------
181
+ # Internals
182
+ # ------------------------------------------------------------------
183
+ def __repr__(self) -> str:
184
+ return f"FaTok(vocab_size={self.vocab_size}, path={self._path.name!s})"
185
+
186
+ def __len__(self) -> int:
187
+ return self.vocab_size
188
+
189
+
190
+ # ---------------------------------------------------------------------------
191
+ # Module-level lazy accessor
192
+ # ---------------------------------------------------------------------------
193
+ @lru_cache(maxsize=1)
194
+ def get_default() -> FaTok:
195
+ """Return a process-wide cached default tokenizer.
196
+
197
+ Useful when you call encode/decode in a hot loop and don't want to
198
+ pay the file-load cost on every call.
199
+ """
200
+ return FaTok()
201
+
202
+
203
+ def info() -> dict:
204
+ """Return metadata about the bundled tokenizer (vocab size, special tokens, path)."""
205
+ stats_path = _DATA_DIR / "training_stats.json"
206
+ if stats_path.exists():
207
+ with open(stats_path, encoding="utf-8") as f:
208
+ stats = json.load(f)
209
+ else:
210
+ stats = {}
211
+ return {
212
+ "path": str(_DEFAULT_TOKENIZER_PATH),
213
+ "vocab_size": FaTok().vocab_size,
214
+ **stats,
215
+ }
@@ -0,0 +1,170 @@
1
+ Metadata-Version: 2.5
2
+ Name: faTok
3
+ Version: 0.1.0
4
+ Summary: A Persian-first BPE tokenizer with built-in Persian normalization (Yeh/Kaf, diacritics, ZWNJ-aware).
5
+ Project-URL: Homepage, https://github.com/your-org/faTok
6
+ Project-URL: Documentation, https://github.com/your-org/faTok#readme
7
+ Project-URL: Issues, https://github.com/your-org/faTok/issues
8
+ Project-URL: Source, https://github.com/your-org/faTok
9
+ Author: faTok contributors
10
+ License: MIT License
11
+
12
+ Copyright (c) 2026 faTok contributors
13
+
14
+ Permission is hereby granted, free of charge, to any person obtaining a copy
15
+ of this software and associated documentation files (the "Software"), to deal
16
+ in the Software without restriction, including without limitation the rights
17
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
18
+ copies of the Software, and to permit persons to whom the Software is
19
+ furnished to do so, subject to the following conditions:
20
+
21
+ The above copyright notice and this permission notice shall be included in all
22
+ copies or substantial portions of the Software.
23
+
24
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
25
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
26
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
27
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
28
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
29
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30
+ SOFTWARE.
31
+ License-File: LICENSE
32
+ Keywords: bpe,byte-level,farsi,huggingface,nlp,persian,subword,tokenizer
33
+ Classifier: Development Status :: 4 - Beta
34
+ Classifier: Intended Audience :: Developers
35
+ Classifier: Intended Audience :: Science/Research
36
+ Classifier: License :: OSI Approved :: MIT License
37
+ Classifier: Natural Language :: Persian
38
+ Classifier: Operating System :: OS Independent
39
+ Classifier: Programming Language :: Python :: 3
40
+ Classifier: Programming Language :: Python :: 3 :: Only
41
+ Classifier: Programming Language :: Python :: 3.9
42
+ Classifier: Programming Language :: Python :: 3.10
43
+ Classifier: Programming Language :: Python :: 3.11
44
+ Classifier: Programming Language :: Python :: 3.12
45
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
46
+ Classifier: Topic :: Text Processing :: Linguistic
47
+ Requires-Python: >=3.9
48
+ Requires-Dist: pyarrow<19,>=15
49
+ Requires-Dist: tokenizers<0.21,>=0.20
50
+ Requires-Dist: transformers<5,>=4.46
51
+ Provides-Extra: dev
52
+ Requires-Dist: build>=1.0; extra == 'dev'
53
+ Requires-Dist: pytest>=7.0; extra == 'dev'
54
+ Requires-Dist: twine>=4.0; extra == 'dev'
55
+ Description-Content-Type: text/markdown
56
+
57
+ # faTok
58
+
59
+ **faTok** — *Persian-first BPE tokenizer.*
60
+
61
+ A small, dependency-light Python package that ships a byte-level BPE
62
+ tokenizer with built-in Persian normalization:
63
+
64
+ - Arabic Yeh / Keheh normalization (`ي → ی`, `ك → ک`, `ى → ی`)
65
+ - Stripping of Arabic diacritics (`tashkeel`) and Tatweel (`ـ`)
66
+ - ZWNJ (نیم‌فاصله) preserved as part of words
67
+ - GPT-2 style pre-tokenization adapted for Persian + ZWNJ
68
+
69
+ The bundled tokenizer is small (≈ 4K vocab) so the package is usable
70
+ immediately. Drop your own `tokenizer.json` into
71
+ `site-packages/fatok/data/` to upgrade to a larger vocab — no code
72
+ changes required.
73
+
74
+ ## Installation
75
+
76
+ ```bash
77
+ pip install fatok
78
+ ```
79
+
80
+ For local development:
81
+
82
+ ```bash
83
+ git clone https://github.com/your-org/faTok
84
+ cd faTok
85
+ pip install -e .
86
+ ```
87
+
88
+ ## Quick start
89
+
90
+ ```python
91
+ from fatok import FaTok
92
+
93
+ tok = FaTok()
94
+ ids = tok.encode("سلام دنیا، حالت چطوره؟")
95
+ print(ids) # [123, 456, 789, ...]
96
+ print(tok.decode(ids)) # سلام دنیا، حالت چطوره؟
97
+ print(tok.vocab_size) # 4096 (bundled) or whatever you replaced it with
98
+ print(tok.count("یک متن")) # number of tokens
99
+ print(tok.normalize("سَلام كِيف حالُك")) # سلام کیف حالک
100
+ ```
101
+
102
+ ### Batch encode / decode
103
+
104
+ ```python
105
+ ids_batch = tok.encode_batch(["جمله‌ی اول", "جمله‌ی دوم"])
106
+ texts = tok.decode_batch(ids_batch)
107
+ ```
108
+
109
+ ### Add special tokens
110
+
111
+ ```python
112
+ ids = tok.encode("### سوال: پایتخت ایران؟", add_special_tokens=True)
113
+ # ids starts with <s> and ends with </s>
114
+ ```
115
+
116
+ ### Use the underlying HuggingFace tokenizer
117
+
118
+ ```python
119
+ hf = tok.to_hf(model_max_length=4096)
120
+ # hf is a transformers.PreTrainedTokenizerFast — pass to any HF model
121
+ ```
122
+
123
+ ## Command-line interface
124
+
125
+ ```bash
126
+ fatok info
127
+ fatok encode "سلام دنیا"
128
+ fatok encode "سلام دنیا" --json
129
+ fatok decode 123 456 789
130
+ fatok count "یک متن نسبتاً طولانی برای شمارش توکن‌ها"
131
+ fatok normalize "سَلام كَيْف حالُك"
132
+ ```
133
+
134
+ ## Using your own tokenizer
135
+
136
+ After training your own Persian BPE tokenizer (for example with the
137
+ `train_persian_tokenizer_v2.ipynb` notebook), replace the bundled one:
138
+
139
+ ```bash
140
+ cp /path/to/your/tokenizer.json "$(python -c 'import fatok, os; print(os.path.dirname(fatok.__file__))')/data/tokenizer.json"
141
+ ```
142
+
143
+ Then re-import `fatok` and the new vocab is live.
144
+
145
+ ## API reference
146
+
147
+ | Method | Returns | Description |
148
+ |---|---|---|
149
+ | `FaTok(path=None)` | `FaTok` | Load the bundled (or custom) tokenizer. |
150
+ | `tok.encode(text, add_special_tokens=False)` | `list[int]` | Encode a string. |
151
+ | `tok.encode_batch(texts, ...)` | `list[list[int]]` | Encode many strings. |
152
+ | `tok.decode(ids, skip_special_tokens=True)` | `str` | Decode ids. |
153
+ | `tok.decode_batch(batch_ids, ...)` | `list[str]` | Decode many. |
154
+ | `tok.normalize(text)` | `str` | Apply Persian normalization only. |
155
+ | `tok.count(text)` | `int` | Token count (cheap). |
156
+ | `tok.token_to_id(tok_str)` | `int` | `-1` if absent. |
157
+ | `tok.id_to_token(idx)` | `str \| None` | Reverse lookup. |
158
+ | `tok.vocab_size` | `int` | Including specials. |
159
+ | `tok.vocab` | `dict` | Token → id map. |
160
+ | `tok.to_hf(**kw)` | `PreTrainedTokenizerFast` | For HF ecosystem. |
161
+
162
+ ## Versioning
163
+
164
+ This package follows the bundled tokenizer's vocab size: any change to
165
+ the bundled `tokenizer.json` (different vocab, different special tokens)
166
+ requires a minor-version bump.
167
+
168
+ ## License
169
+
170
+ MIT — see [LICENSE](LICENSE).
@@ -0,0 +1,13 @@
1
+ fatok/__about__.py,sha256=yY1rqFv7FPcMDWUGaIOeH0rDQGqHcZgFd_97e5Wz2i8,241
2
+ fatok/__init__.py,sha256=rtdmUoS5wf7l5OPkqFPmM9QK2J6RxRjKG0QQjSSHolw,474
3
+ fatok/cli.py,sha256=uNaHaQh7PC7CegHR_mq6Y7NlvMTiDotIUenZHXny_wM,3009
4
+ fatok/tokenizer.py,sha256=FeUwSOlofH3ATEAa49AnHvnvN__4nzRT8Jta-Fkt8GI,7861
5
+ fatok/data/special_tokens_map.json,sha256=gDD19ozZic9dh0X1-Eugt3VSc6-RjF6hnoz3Y7V5o8Y,151
6
+ fatok/data/tokenizer.json,sha256=1SKhFuwl7Q3iH62EESCta3W3_kVNscUqY0cS-sWRbMk,34493
7
+ fatok/data/tokenizer_config.json,sha256=D6WE2qW2_L8QIDAulH-Z8Qz0Ws3kKELVqoWBjL2gYYY,1169
8
+ fatok/data/training_stats.json,sha256=5Ep-TLuJcmru0LkyNZ1m6CB-a4pFaxSi-2YrIrgXxLA,299
9
+ fatok-0.1.0.dist-info/METADATA,sha256=Ow_7r-Oj6q4l5FyKYgMjkWv7VuspQdlSoKRMHlnU8-s,6270
10
+ fatok-0.1.0.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
11
+ fatok-0.1.0.dist-info/entry_points.txt,sha256=eAUj82uSa99g31p7CvRXn5No4n2xmP9Q4nHb9th3jKQ,41
12
+ fatok-0.1.0.dist-info/licenses/LICENSE,sha256=BwFgikNxgXi5R4waukXsziu53qbxJbUU9QcvQs9B97U,1075
13
+ fatok-0.1.0.dist-info/RECORD,,
@@ -0,0 +1,4 @@
1
+ Wheel-Version: 1.0
2
+ Generator: hatchling 1.33.0
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
@@ -0,0 +1,2 @@
1
+ [console_scripts]
2
+ fatok = fatok.cli:main
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 faTok contributors
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.