faTok 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
fatok-0.1.0/.gitignore ADDED
@@ -0,0 +1,2 @@
1
+ skills/
2
+ node_modules/
fatok-0.1.0/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 faTok contributors
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
fatok-0.1.0/PKG-INFO ADDED
@@ -0,0 +1,170 @@
1
+ Metadata-Version: 2.5
2
+ Name: faTok
3
+ Version: 0.1.0
4
+ Summary: A Persian-first BPE tokenizer with built-in Persian normalization (Yeh/Kaf, diacritics, ZWNJ-aware).
5
+ Project-URL: Homepage, https://github.com/your-org/faTok
6
+ Project-URL: Documentation, https://github.com/your-org/faTok#readme
7
+ Project-URL: Issues, https://github.com/your-org/faTok/issues
8
+ Project-URL: Source, https://github.com/your-org/faTok
9
+ Author: faTok contributors
10
+ License: MIT License
11
+
12
+ Copyright (c) 2026 faTok contributors
13
+
14
+ Permission is hereby granted, free of charge, to any person obtaining a copy
15
+ of this software and associated documentation files (the "Software"), to deal
16
+ in the Software without restriction, including without limitation the rights
17
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
18
+ copies of the Software, and to permit persons to whom the Software is
19
+ furnished to do so, subject to the following conditions:
20
+
21
+ The above copyright notice and this permission notice shall be included in all
22
+ copies or substantial portions of the Software.
23
+
24
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
25
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
26
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
27
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
28
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
29
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30
+ SOFTWARE.
31
+ License-File: LICENSE
32
+ Keywords: bpe,byte-level,farsi,huggingface,nlp,persian,subword,tokenizer
33
+ Classifier: Development Status :: 4 - Beta
34
+ Classifier: Intended Audience :: Developers
35
+ Classifier: Intended Audience :: Science/Research
36
+ Classifier: License :: OSI Approved :: MIT License
37
+ Classifier: Natural Language :: Persian
38
+ Classifier: Operating System :: OS Independent
39
+ Classifier: Programming Language :: Python :: 3
40
+ Classifier: Programming Language :: Python :: 3 :: Only
41
+ Classifier: Programming Language :: Python :: 3.9
42
+ Classifier: Programming Language :: Python :: 3.10
43
+ Classifier: Programming Language :: Python :: 3.11
44
+ Classifier: Programming Language :: Python :: 3.12
45
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
46
+ Classifier: Topic :: Text Processing :: Linguistic
47
+ Requires-Python: >=3.9
48
+ Requires-Dist: pyarrow<19,>=15
49
+ Requires-Dist: tokenizers<0.21,>=0.20
50
+ Requires-Dist: transformers<5,>=4.46
51
+ Provides-Extra: dev
52
+ Requires-Dist: build>=1.0; extra == 'dev'
53
+ Requires-Dist: pytest>=7.0; extra == 'dev'
54
+ Requires-Dist: twine>=4.0; extra == 'dev'
55
+ Description-Content-Type: text/markdown
56
+
57
+ # faTok
58
+
59
+ **faTok** — *Persian-first BPE tokenizer.*
60
+
61
+ A small, dependency-light Python package that ships a byte-level BPE
62
+ tokenizer with built-in Persian normalization:
63
+
64
+ - Arabic Yeh / Keheh normalization (`ي → ی`, `ك → ک`, `ى → ی`)
65
+ - Stripping of Arabic diacritics (`tashkeel`) and Tatweel (`ـ`)
66
+ - ZWNJ (نیم‌فاصله) preserved as part of words
67
+ - GPT-2 style pre-tokenization adapted for Persian + ZWNJ
68
+
69
+ The bundled tokenizer is small (≈ 4K vocab) so the package is usable
70
+ immediately. Drop your own `tokenizer.json` into
71
+ `site-packages/fatok/data/` to upgrade to a larger vocab — no code
72
+ changes required.
73
+
74
+ ## Installation
75
+
76
+ ```bash
77
+ pip install fatok
78
+ ```
79
+
80
+ For local development:
81
+
82
+ ```bash
83
+ git clone https://github.com/your-org/faTok
84
+ cd faTok
85
+ pip install -e .
86
+ ```
87
+
88
+ ## Quick start
89
+
90
+ ```python
91
+ from fatok import FaTok
92
+
93
+ tok = FaTok()
94
+ ids = tok.encode("سلام دنیا، حالت چطوره؟")
95
+ print(ids) # [123, 456, 789, ...]
96
+ print(tok.decode(ids)) # سلام دنیا، حالت چطوره؟
97
+ print(tok.vocab_size) # 4096 (bundled) or whatever you replaced it with
98
+ print(tok.count("یک متن")) # number of tokens
99
+ print(tok.normalize("سَلام كِيف حالُك")) # سلام کیف حالک
100
+ ```
101
+
102
+ ### Batch encode / decode
103
+
104
+ ```python
105
+ ids_batch = tok.encode_batch(["جمله‌ی اول", "جمله‌ی دوم"])
106
+ texts = tok.decode_batch(ids_batch)
107
+ ```
108
+
109
+ ### Add special tokens
110
+
111
+ ```python
112
+ ids = tok.encode("### سوال: پایتخت ایران؟", add_special_tokens=True)
113
+ # ids starts with <s> and ends with </s>
114
+ ```
115
+
116
+ ### Use the underlying HuggingFace tokenizer
117
+
118
+ ```python
119
+ hf = tok.to_hf(model_max_length=4096)
120
+ # hf is a transformers.PreTrainedTokenizerFast — pass to any HF model
121
+ ```
122
+
123
+ ## Command-line interface
124
+
125
+ ```bash
126
+ fatok info
127
+ fatok encode "سلام دنیا"
128
+ fatok encode "سلام دنیا" --json
129
+ fatok decode 123 456 789
130
+ fatok count "یک متن نسبتاً طولانی برای شمارش توکن‌ها"
131
+ fatok normalize "سَلام كَيْف حالُك"
132
+ ```
133
+
134
+ ## Using your own tokenizer
135
+
136
+ After training your own Persian BPE tokenizer (for example with the
137
+ `train_persian_tokenizer_v2.ipynb` notebook), replace the bundled one:
138
+
139
+ ```bash
140
+ cp /path/to/your/tokenizer.json "$(python -c 'import fatok, os; print(os.path.dirname(fatok.__file__))')/data/tokenizer.json"
141
+ ```
142
+
143
+ Then re-import `fatok` and the new vocab is live.
144
+
145
+ ## API reference
146
+
147
+ | Method | Returns | Description |
148
+ |---|---|---|
149
+ | `FaTok(path=None)` | `FaTok` | Load the bundled (or custom) tokenizer. |
150
+ | `tok.encode(text, add_special_tokens=False)` | `list[int]` | Encode a string. |
151
+ | `tok.encode_batch(texts, ...)` | `list[list[int]]` | Encode many strings. |
152
+ | `tok.decode(ids, skip_special_tokens=True)` | `str` | Decode ids. |
153
+ | `tok.decode_batch(batch_ids, ...)` | `list[str]` | Decode many. |
154
+ | `tok.normalize(text)` | `str` | Apply Persian normalization only. |
155
+ | `tok.count(text)` | `int` | Token count (cheap). |
156
+ | `tok.token_to_id(tok_str)` | `int` | `-1` if absent. |
157
+ | `tok.id_to_token(idx)` | `str \| None` | Reverse lookup. |
158
+ | `tok.vocab_size` | `int` | Including specials. |
159
+ | `tok.vocab` | `dict` | Token → id map. |
160
+ | `tok.to_hf(**kw)` | `PreTrainedTokenizerFast` | For HF ecosystem. |
161
+
162
+ ## Versioning
163
+
164
+ This package follows the bundled tokenizer's vocab size: any change to
165
+ the bundled `tokenizer.json` (different vocab, different special tokens)
166
+ requires a minor-version bump.
167
+
168
+ ## License
169
+
170
+ MIT — see [LICENSE](LICENSE).
fatok-0.1.0/README.md ADDED
@@ -0,0 +1,114 @@
1
+ # faTok
2
+
3
+ **faTok** — *Persian-first BPE tokenizer.*
4
+
5
+ A small, dependency-light Python package that ships a byte-level BPE
6
+ tokenizer with built-in Persian normalization:
7
+
8
+ - Arabic Yeh / Keheh normalization (`ي → ی`, `ك → ک`, `ى → ی`)
9
+ - Stripping of Arabic diacritics (`tashkeel`) and Tatweel (`ـ`)
10
+ - ZWNJ (نیم‌فاصله) preserved as part of words
11
+ - GPT-2 style pre-tokenization adapted for Persian + ZWNJ
12
+
13
+ The bundled tokenizer is small (≈ 4K vocab) so the package is usable
14
+ immediately. Drop your own `tokenizer.json` into
15
+ `site-packages/fatok/data/` to upgrade to a larger vocab — no code
16
+ changes required.
17
+
18
+ ## Installation
19
+
20
+ ```bash
21
+ pip install fatok
22
+ ```
23
+
24
+ For local development:
25
+
26
+ ```bash
27
+ git clone https://github.com/your-org/faTok
28
+ cd faTok
29
+ pip install -e .
30
+ ```
31
+
32
+ ## Quick start
33
+
34
+ ```python
35
+ from fatok import FaTok
36
+
37
+ tok = FaTok()
38
+ ids = tok.encode("سلام دنیا، حالت چطوره؟")
39
+ print(ids) # [123, 456, 789, ...]
40
+ print(tok.decode(ids)) # سلام دنیا، حالت چطوره؟
41
+ print(tok.vocab_size) # 4096 (bundled) or whatever you replaced it with
42
+ print(tok.count("یک متن")) # number of tokens
43
+ print(tok.normalize("سَلام كِيف حالُك")) # سلام کیف حالک
44
+ ```
45
+
46
+ ### Batch encode / decode
47
+
48
+ ```python
49
+ ids_batch = tok.encode_batch(["جمله‌ی اول", "جمله‌ی دوم"])
50
+ texts = tok.decode_batch(ids_batch)
51
+ ```
52
+
53
+ ### Add special tokens
54
+
55
+ ```python
56
+ ids = tok.encode("### سوال: پایتخت ایران؟", add_special_tokens=True)
57
+ # ids starts with <s> and ends with </s>
58
+ ```
59
+
60
+ ### Use the underlying HuggingFace tokenizer
61
+
62
+ ```python
63
+ hf = tok.to_hf(model_max_length=4096)
64
+ # hf is a transformers.PreTrainedTokenizerFast — pass to any HF model
65
+ ```
66
+
67
+ ## Command-line interface
68
+
69
+ ```bash
70
+ fatok info
71
+ fatok encode "سلام دنیا"
72
+ fatok encode "سلام دنیا" --json
73
+ fatok decode 123 456 789
74
+ fatok count "یک متن نسبتاً طولانی برای شمارش توکن‌ها"
75
+ fatok normalize "سَلام كَيْف حالُك"
76
+ ```
77
+
78
+ ## Using your own tokenizer
79
+
80
+ After training your own Persian BPE tokenizer (for example with the
81
+ `train_persian_tokenizer_v2.ipynb` notebook), replace the bundled one:
82
+
83
+ ```bash
84
+ cp /path/to/your/tokenizer.json "$(python -c 'import fatok, os; print(os.path.dirname(fatok.__file__))')/data/tokenizer.json"
85
+ ```
86
+
87
+ Then re-import `fatok` and the new vocab is live.
88
+
89
+ ## API reference
90
+
91
+ | Method | Returns | Description |
92
+ |---|---|---|
93
+ | `FaTok(path=None)` | `FaTok` | Load the bundled (or custom) tokenizer. |
94
+ | `tok.encode(text, add_special_tokens=False)` | `list[int]` | Encode a string. |
95
+ | `tok.encode_batch(texts, ...)` | `list[list[int]]` | Encode many strings. |
96
+ | `tok.decode(ids, skip_special_tokens=True)` | `str` | Decode ids. |
97
+ | `tok.decode_batch(batch_ids, ...)` | `list[str]` | Decode many. |
98
+ | `tok.normalize(text)` | `str` | Apply Persian normalization only. |
99
+ | `tok.count(text)` | `int` | Token count (cheap). |
100
+ | `tok.token_to_id(tok_str)` | `int` | `-1` if absent. |
101
+ | `tok.id_to_token(idx)` | `str \| None` | Reverse lookup. |
102
+ | `tok.vocab_size` | `int` | Including specials. |
103
+ | `tok.vocab` | `dict` | Token → id map. |
104
+ | `tok.to_hf(**kw)` | `PreTrainedTokenizerFast` | For HF ecosystem. |
105
+
106
+ ## Versioning
107
+
108
+ This package follows the bundled tokenizer's vocab size: any change to
109
+ the bundled `tokenizer.json` (different vocab, different special tokens)
110
+ requires a minor-version bump.
111
+
112
+ ## License
113
+
114
+ MIT — see [LICENSE](LICENSE).
@@ -0,0 +1,69 @@
1
+ [build-system]
2
+ requires = ["hatchling>=1.21"]
3
+ build-backend = "hatchling.build"
4
+
5
+ [project]
6
+ name = "faTok"
7
+ version = "0.1.0"
8
+ description = "A Persian-first BPE tokenizer with built-in Persian normalization (Yeh/Kaf, diacritics, ZWNJ-aware)."
9
+ readme = "README.md"
10
+ requires-python = ">=3.9"
11
+ license = { file = "LICENSE" }
12
+ authors = [{ name = "faTok contributors" }]
13
+ keywords = [
14
+ "persian", "farsi", "tokenizer", "bpe", "byte-level",
15
+ "nlp", "huggingface", "subword",
16
+ ]
17
+ classifiers = [
18
+ "Development Status :: 4 - Beta",
19
+ "Intended Audience :: Developers",
20
+ "Intended Audience :: Science/Research",
21
+ "License :: OSI Approved :: MIT License",
22
+ "Natural Language :: Persian",
23
+ "Operating System :: OS Independent",
24
+ "Programming Language :: Python :: 3",
25
+ "Programming Language :: Python :: 3 :: Only",
26
+ "Programming Language :: Python :: 3.9",
27
+ "Programming Language :: Python :: 3.10",
28
+ "Programming Language :: Python :: 3.11",
29
+ "Programming Language :: Python :: 3.12",
30
+ "Topic :: Scientific/Engineering :: Artificial Intelligence",
31
+ "Topic :: Text Processing :: Linguistic",
32
+ ]
33
+ dependencies = [
34
+ # Same pinned HF stack verified in requirements.txt — these are the
35
+ # versions known to import cleanly together with numpy 1.x and 2.x.
36
+ "tokenizers>=0.20,<0.21",
37
+ "transformers>=4.46,<5",
38
+ # pyarrow<19 is required when numpy<2 is in the user's environment;
39
+ # pyarrow 18.x supports BOTH numpy 1.x and 2.x so it's safe to pin here.
40
+ "pyarrow>=15,<19",
41
+ ]
42
+
43
+ [project.optional-dependencies]
44
+ dev = [
45
+ "pytest>=7.0",
46
+ "build>=1.0",
47
+ "twine>=4.0",
48
+ ]
49
+
50
+ [project.urls]
51
+ Homepage = "https://github.com/your-org/faTok"
52
+ Documentation = "https://github.com/your-org/faTok#readme"
53
+ Issues = "https://github.com/your-org/faTok/issues"
54
+ Source = "https://github.com/your-org/faTok"
55
+
56
+ [project.scripts]
57
+ fatok = "fatok.cli:main"
58
+
59
+ [tool.hatch.build.targets.wheel]
60
+ packages = ["src/fatok"]
61
+
62
+ [tool.hatch.build.targets.sdist]
63
+ include = [
64
+ "src/fatok",
65
+ "tests",
66
+ "README.md",
67
+ "LICENSE",
68
+ "pyproject.toml",
69
+ ]
@@ -0,0 +1,7 @@
1
+ """Single source of truth for the faTok version.
2
+
3
+ Kept in its own module so that ``setup.py`` / ``pyproject.toml``-less tools
4
+ can read it without importing the package (which would force the
5
+ ``tokenizers`` import).
6
+ """
7
+ __version__ = "0.1.0"
@@ -0,0 +1,18 @@
1
+ """faTok — a Persian-first BPE tokenizer.
2
+
3
+ Public API
4
+ ----------
5
+ from fatok import FaTok
6
+
7
+ tok = FaTok() # load the bundled Persian tokenizer
8
+ ids = tok.encode("سلام دنیا")
9
+ text = tok.decode(ids)
10
+ n = tok.vocab_size
11
+ hf = tok.to_hf() # underlying PreTrainedTokenizerFast
12
+
13
+ See ``FaTok`` for the full API.
14
+ """
15
+ from .tokenizer import FaTok
16
+ from .__about__ import __version__
17
+
18
+ __all__ = ["FaTok", "__version__"]
@@ -0,0 +1,95 @@
1
+ """``fatok`` command-line interface.
2
+
3
+ Examples
4
+ --------
5
+ fatok info
6
+ fatok encode "سلام دنیا"
7
+ fatok encode "سلام دنیا" --json
8
+ fatok decode 1234 5678 910
9
+ fatok count "یک متن نسبتا طولانی برای شمارش توکن‌ها"
10
+ fatok normalize "سَلام كَيْف حالُك"
11
+ """
12
+ from __future__ import annotations
13
+
14
+ import argparse
15
+ import json
16
+ import sys
17
+ from typing import List, Optional
18
+
19
+ from .tokenizer import FaTok, info
20
+
21
+
22
+ def _build_parser() -> argparse.ArgumentParser:
23
+ p = argparse.ArgumentParser(
24
+ prog="fatok",
25
+ description="faTok — Persian BPE tokenizer.",
26
+ )
27
+ sub = p.add_subparsers(dest="cmd", required=True)
28
+
29
+ sub.add_parser("info", help="Print metadata about the bundled tokenizer.")
30
+
31
+ enc = sub.add_parser("encode", help="Encode text into token ids.")
32
+ enc.add_argument("text", help="Text to encode.")
33
+ enc.add_argument("--special-tokens", action="store_true",
34
+ help="Wrap with <s> ... </s>.")
35
+ enc.add_argument("--json", action="store_true",
36
+ help="Output JSON: {ids, n, text}.")
37
+ enc.add_argument("--path", help="Custom tokenizer.json path.")
38
+
39
+ dec = sub.add_parser("decode", help="Decode token ids back into text.")
40
+ dec.add_argument("ids", nargs="+", type=int, help="Token ids to decode.")
41
+ dec.add_argument("--keep-special-tokens", action="store_true",
42
+ help="Do not skip special tokens in the output.")
43
+ dec.add_argument("--path", help="Custom tokenizer.json path.")
44
+
45
+ cnt = sub.add_parser("count", help="Count tokens for a text.")
46
+ cnt.add_argument("text", help="Text to count tokens for.")
47
+ cnt.add_argument("--path", help="Custom tokenizer.json path.")
48
+
49
+ norm = sub.add_parser("normalize", help="Apply Persian normalization only.")
50
+ norm.add_argument("text", help="Text to normalize.")
51
+ norm.add_argument("--path", help="Custom tokenizer.json path.")
52
+
53
+ return p
54
+
55
+
56
+ def main(argv: Optional[List[str]] = None) -> int:
57
+ args = _build_parser().parse_args(argv)
58
+
59
+ if args.cmd == "info":
60
+ data = info()
61
+ print(json.dumps(data, ensure_ascii=False, indent=2))
62
+ return 0
63
+
64
+ tok = FaTok(path=getattr(args, "path", None))
65
+
66
+ if args.cmd == "encode":
67
+ ids = tok.encode(args.text, add_special_tokens=args.special_tokens)
68
+ if args.json:
69
+ print(json.dumps({
70
+ "ids": ids,
71
+ "n": len(ids),
72
+ "text": args.text,
73
+ }, ensure_ascii=False))
74
+ else:
75
+ print(" ".join(str(i) for i in ids))
76
+ return 0
77
+
78
+ if args.cmd == "decode":
79
+ text = tok.decode(args.ids, skip_special_tokens=not args.keep_special_tokens)
80
+ print(text)
81
+ return 0
82
+
83
+ if args.cmd == "count":
84
+ print(tok.count(args.text))
85
+ return 0
86
+
87
+ if args.cmd == "normalize":
88
+ print(tok.normalize(args.text))
89
+ return 0
90
+
91
+ return 1
92
+
93
+
94
+ if __name__ == "__main__":
95
+ sys.exit(main())
@@ -0,0 +1,9 @@
1
+ {
2
+ "additional_special_tokens": [
3
+ "### سوال:",
4
+ "### پاسخ:"
5
+ ],
6
+ "bos_token": "<s>",
7
+ "eos_token": "</s>",
8
+ "pad_token": "<pad>"
9
+ }