ichkil 1.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ichkil-1.0.0/.github/workflows/ci.yml +88 -0
- ichkil-1.0.0/.github/workflows/publish.yml +31 -0
- ichkil-1.0.0/.gitignore +31 -0
- ichkil-1.0.0/CHANGELOG.md +29 -0
- ichkil-1.0.0/LICENSE +21 -0
- ichkil-1.0.0/PKG-INFO +173 -0
- ichkil-1.0.0/README.md +139 -0
- ichkil-1.0.0/examples/quickstart.py +37 -0
- ichkil-1.0.0/pyproject.toml +79 -0
- ichkil-1.0.0/src/ichkil/__init__.py +103 -0
- ichkil-1.0.0/src/ichkil/cli.py +73 -0
- ichkil-1.0.0/src/ichkil/diacritizer.py +149 -0
- ichkil-1.0.0/src/ichkil/download.py +157 -0
- ichkil-1.0.0/src/ichkil/errors.py +21 -0
- ichkil-1.0.0/src/ichkil/pipeline.py +148 -0
- ichkil-1.0.0/src/ichkil/py.typed +0 -0
- ichkil-1.0.0/src/ichkil/version.py +3 -0
- ichkil-1.0.0/tests/golden.json +1558 -0
- ichkil-1.0.0/tests/test_diacritizer.py +150 -0
- ichkil-1.0.0/tests/test_pipeline.py +143 -0
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main]
|
|
6
|
+
pull_request:
|
|
7
|
+
|
|
8
|
+
concurrency:
|
|
9
|
+
group: ci-${{ github.workflow }}-${{ github.ref }}
|
|
10
|
+
cancel-in-progress: true
|
|
11
|
+
|
|
12
|
+
jobs:
|
|
13
|
+
lint:
|
|
14
|
+
runs-on: ubuntu-latest
|
|
15
|
+
steps:
|
|
16
|
+
- uses: actions/checkout@v4
|
|
17
|
+
- uses: actions/setup-python@v5
|
|
18
|
+
with:
|
|
19
|
+
python-version: "3.12"
|
|
20
|
+
- name: Install
|
|
21
|
+
run: |
|
|
22
|
+
python -m pip install --upgrade pip
|
|
23
|
+
pip install ruff
|
|
24
|
+
- name: Ruff (lint)
|
|
25
|
+
run: ruff check src tests examples
|
|
26
|
+
- name: Ruff (format check)
|
|
27
|
+
run: ruff format --check src tests examples
|
|
28
|
+
|
|
29
|
+
test:
|
|
30
|
+
runs-on: ubuntu-latest
|
|
31
|
+
strategy:
|
|
32
|
+
fail-fast: false
|
|
33
|
+
matrix:
|
|
34
|
+
python-version: ["3.9", "3.10", "3.11", "3.12"]
|
|
35
|
+
steps:
|
|
36
|
+
- uses: actions/checkout@v4
|
|
37
|
+
- uses: actions/setup-python@v5
|
|
38
|
+
with:
|
|
39
|
+
python-version: ${{ matrix.python-version }}
|
|
40
|
+
cache: pip
|
|
41
|
+
- name: Install
|
|
42
|
+
run: |
|
|
43
|
+
python -m pip install --upgrade pip
|
|
44
|
+
pip install -e ".[dev]"
|
|
45
|
+
- name: Test (pure pipeline — no network)
|
|
46
|
+
run: pytest -m "not integration" --cov --cov-report=xml
|
|
47
|
+
- name: Test (integration — downloads model once, ~5 MB)
|
|
48
|
+
run: pytest -m integration
|
|
49
|
+
env:
|
|
50
|
+
HF_HUB_DISABLE_TELEMETRY: "1"
|
|
51
|
+
|
|
52
|
+
build:
|
|
53
|
+
runs-on: ubuntu-latest
|
|
54
|
+
needs: [lint, test]
|
|
55
|
+
steps:
|
|
56
|
+
- uses: actions/checkout@v4
|
|
57
|
+
- uses: actions/setup-python@v5
|
|
58
|
+
with:
|
|
59
|
+
python-version: "3.12"
|
|
60
|
+
- name: Install build frontend
|
|
61
|
+
run: python -m pip install --upgrade pip build
|
|
62
|
+
- name: Build sdist + wheel
|
|
63
|
+
run: python -m build
|
|
64
|
+
- name: Verify wheel metadata
|
|
65
|
+
run: |
|
|
66
|
+
python - <<'PY'
|
|
67
|
+
import glob, zipfile
|
|
68
|
+
whl = sorted(glob.glob("dist/*.whl"))[-1]
|
|
69
|
+
with zipfile.ZipFile(whl) as z:
|
|
70
|
+
names = z.namelist()
|
|
71
|
+
assert any(n.startswith("ichkil/") for n in names), "package dir missing"
|
|
72
|
+
assert "ichkil/py.typed" in names, "py.typed marker missing"
|
|
73
|
+
print("wheel OK:", whl)
|
|
74
|
+
PY
|
|
75
|
+
- name: Verify sdist
|
|
76
|
+
run: |
|
|
77
|
+
python - <<'PY'
|
|
78
|
+
import glob, tarfile
|
|
79
|
+
sdist = sorted(glob.glob("dist/*.tar.gz"))[-1]
|
|
80
|
+
with tarfile.open(sdist) as t:
|
|
81
|
+
names = t.getnames()
|
|
82
|
+
assert any(n.endswith("pyproject.toml") for n in names)
|
|
83
|
+
print("sdist OK:", sdist)
|
|
84
|
+
PY
|
|
85
|
+
- uses: actions/upload-artifact@v4
|
|
86
|
+
with:
|
|
87
|
+
name: dist
|
|
88
|
+
path: dist/
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
name: Publish
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
tags:
|
|
6
|
+
- "v*"
|
|
7
|
+
|
|
8
|
+
concurrency:
|
|
9
|
+
group: publish-${{ github.ref }}
|
|
10
|
+
cancel-in-progress: false
|
|
11
|
+
|
|
12
|
+
permissions:
|
|
13
|
+
contents: read
|
|
14
|
+
id-token: write # required for trusted publishing (OIDC)
|
|
15
|
+
|
|
16
|
+
jobs:
|
|
17
|
+
publish:
|
|
18
|
+
runs-on: ubuntu-latest
|
|
19
|
+
steps:
|
|
20
|
+
- uses: actions/checkout@v4
|
|
21
|
+
- uses: actions/setup-python@v5
|
|
22
|
+
with:
|
|
23
|
+
python-version: "3.12"
|
|
24
|
+
- name: Install build frontend
|
|
25
|
+
run: python -m pip install --upgrade pip build
|
|
26
|
+
- name: Build sdist + wheel
|
|
27
|
+
run: python -m build
|
|
28
|
+
- name: Publish to PyPI
|
|
29
|
+
uses: pypa/gh-action-pypi-publish@release/v1
|
|
30
|
+
with:
|
|
31
|
+
trusted_publishing: true
|
ichkil-1.0.0/.gitignore
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
# Byte-compiled / optimized files
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
*$py.class
|
|
5
|
+
|
|
6
|
+
# Virtual environments
|
|
7
|
+
.venv/
|
|
8
|
+
venv/
|
|
9
|
+
|
|
10
|
+
# Build & distribution
|
|
11
|
+
build/
|
|
12
|
+
dist/
|
|
13
|
+
*.egg-info/
|
|
14
|
+
*.egg
|
|
15
|
+
|
|
16
|
+
# Test / coverage artifacts
|
|
17
|
+
.pytest_cache/
|
|
18
|
+
.coverage
|
|
19
|
+
.coverage.*
|
|
20
|
+
htmlcov/
|
|
21
|
+
coverage.xml
|
|
22
|
+
|
|
23
|
+
# Tooling caches
|
|
24
|
+
.ruff_cache/
|
|
25
|
+
.mypy_cache/
|
|
26
|
+
.dmypy.json
|
|
27
|
+
|
|
28
|
+
# Editors / OS
|
|
29
|
+
.idea/
|
|
30
|
+
.vscode/
|
|
31
|
+
.DS_Store
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to the `ichkil` Python package are documented here.
|
|
4
|
+
|
|
5
|
+
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
|
6
|
+
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
|
+
|
|
8
|
+
## [1.0.0] - 2026-01-01
|
|
9
|
+
|
|
10
|
+
### Added
|
|
11
|
+
|
|
12
|
+
- Initial public release.
|
|
13
|
+
- `Diacritizer` — one-call Arabic Tashkeel backed by a ≈5 MB ONNX model
|
|
14
|
+
(`input_ids` int64 `[batch, seq]` → `logits` float32 `[batch, seq, 13]`),
|
|
15
|
+
downloaded from Hugging Face (`ichkil/ichkil`), SHA-256-verified, cached
|
|
16
|
+
locally; `Diacritizer.from_file()` for fully offline use.
|
|
17
|
+
- Module-level convenience API: `ichkil.diacritize(text)`,
|
|
18
|
+
`ichkil.predict_labels(text)`, `ichkil.reset_default()`.
|
|
19
|
+
- Pure text pipeline (`ichkil.pipeline`): `strip_diacritics`, `encode`,
|
|
20
|
+
`decode`, `attach`, label tables (13 labels), tatweel normalization —
|
|
21
|
+
shared with and kept in sync with the JS and Go runtimes.
|
|
22
|
+
- Typed API (`py.typed`), `IchkilError` hierarchy
|
|
23
|
+
(`ChecksumError`, `ModelLoadError`, `InferenceError`).
|
|
24
|
+
- CLI: `ichkil` (positional text or stdin, `--model` offline mode,
|
|
25
|
+
`--json`, `--repo`, `--revision`, `--version`).
|
|
26
|
+
- Cross-runtime golden vector suite (`tests/golden.json`, copied from the
|
|
27
|
+
core repository) plus pure-pipeline and integration tests.
|
|
28
|
+
- CI: lint (ruff), test matrix Python 3.9 – 3.12, sdist + wheel build.
|
|
29
|
+
- PyPI publishing via GitHub trusted publishing (OIDC) on `v*` tags.
|
ichkil-1.0.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Maaouia BenHamed
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
ichkil-1.0.0/PKG-INFO
ADDED
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: ichkil
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: Arabic Tashkeel (diacritization) — a tiny, self-contained ONNX model with a clean Python API
|
|
5
|
+
Project-URL: Homepage, https://github.com/Ichkil/ichkil-python
|
|
6
|
+
Project-URL: Repository, https://github.com/Ichkil/ichkil-python
|
|
7
|
+
Project-URL: Issues, https://github.com/Ichkil/ichkil-python/issues
|
|
8
|
+
Project-URL: Hugging Face, https://huggingface.co/ichkil/ichkil
|
|
9
|
+
Project-URL: Live demo (browser), https://huggingface.co/spaces/ichkil/ichkil-demo
|
|
10
|
+
Author: Maaouia BenHamed
|
|
11
|
+
License: MIT
|
|
12
|
+
License-File: LICENSE
|
|
13
|
+
Keywords: arabic,cpu,diacritics,diacritization,harakat,lightweight,nlp,onnx,onnxruntime,tashkeel
|
|
14
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
15
|
+
Classifier: Intended Audience :: Developers
|
|
16
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
17
|
+
Classifier: Operating System :: OS Independent
|
|
18
|
+
Classifier: Programming Language :: Python :: 3
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
23
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
24
|
+
Classifier: Topic :: Text Processing :: Linguistic
|
|
25
|
+
Requires-Python: >=3.9
|
|
26
|
+
Requires-Dist: huggingface-hub>=0.24
|
|
27
|
+
Requires-Dist: onnxruntime<2,>=1.19
|
|
28
|
+
Provides-Extra: dev
|
|
29
|
+
Requires-Dist: build>=1.2; extra == 'dev'
|
|
30
|
+
Requires-Dist: pytest-cov>=4; extra == 'dev'
|
|
31
|
+
Requires-Dist: pytest>=7; extra == 'dev'
|
|
32
|
+
Requires-Dist: ruff>=0.4; extra == 'dev'
|
|
33
|
+
Description-Content-Type: text/markdown
|
|
34
|
+
|
|
35
|
+
# ichkil — Arabic Tashkeel for Python
|
|
36
|
+
|
|
37
|
+
> Vocalize bare Arabic text with a tiny, self-contained ONNX model (≈5 MB, CPU-only,
|
|
38
|
+
> no GPU, no tokenizer, no server). One call, fully offline after the first run.
|
|
39
|
+
|
|
40
|
+
[](https://github.com/Ichkil/ichkil-python/actions/workflows/ci.yml)
|
|
41
|
+
[](https://pypi.org/project/ichkil/)
|
|
42
|
+
[](https://pypi.org/project/ichkil/)
|
|
43
|
+
[](LICENSE)
|
|
44
|
+
[](https://huggingface.co/ichkil/ichkil)
|
|
45
|
+
[](https://huggingface.co/spaces/ichkil/ichkil-demo)
|
|
46
|
+
|
|
47
|
+
`ichkil` downloads its model from [Hugging Face](https://huggingface.co/ichkil/ichkil)
|
|
48
|
+
on first use, verifies its SHA-256, and runs inference with
|
|
49
|
+
[ONNX Runtime](https://onnxruntime.ai/). After that it is 100 % offline.
|
|
50
|
+
|
|
51
|
+
```python
|
|
52
|
+
import ichkil
|
|
53
|
+
|
|
54
|
+
ichkil.diacritize("محمد قرأ الكتاب في المدرسة")
|
|
55
|
+
# -> 'مُحَمَّدٌ قَرَأَ الْكِتَابَ فِي الْمَدْرَسَةِ'
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
## Install
|
|
59
|
+
|
|
60
|
+
```bash
|
|
61
|
+
pip install ichkil
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
Requires Python ≥ 3.9 (tested on 3.9 – 3.12). Dependencies: `huggingface_hub`,
|
|
65
|
+
`onnxruntime` (CPU).
|
|
66
|
+
|
|
67
|
+
## Quickstart
|
|
68
|
+
|
|
69
|
+
```python
|
|
70
|
+
import ichkil
|
|
71
|
+
|
|
72
|
+
# one-liner (lazy model download on first call, then cached)
|
|
73
|
+
ichkil.diacritize("العلم نور والجهل ظلمة")
|
|
74
|
+
# -> 'الْعِلْمِ نُورٌ وَالْجَهْلُ ظُلْمَةُ'
|
|
75
|
+
|
|
76
|
+
# explicit control
|
|
77
|
+
from ichkil import Diacritizer
|
|
78
|
+
|
|
79
|
+
d = Diacritizer() # downloads + verifies + caches the model
|
|
80
|
+
d.diacritize("القهوة جميلة في الصباح")
|
|
81
|
+
# -> 'الْقَهْوَةِ جَمِيلَةٌ فِي الصَّبَاحِ'
|
|
82
|
+
|
|
83
|
+
labels = d.predict_labels("كتاب") # [4, 2, 1, 4] (one label id per character, 0..12)
|
|
84
|
+
|
|
85
|
+
# fully offline: point at local files
|
|
86
|
+
d = Diacritizer.from_file("./model.onnx")
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
### Command line
|
|
90
|
+
|
|
91
|
+
```bash
|
|
92
|
+
pip install ichkil
|
|
93
|
+
ichkil "محمد قرأ الكتاب"
|
|
94
|
+
# مُحَمَّدٌ قَرَأَ الْكِتَابَ
|
|
95
|
+
|
|
96
|
+
ichkil --json "العلم نور"
|
|
97
|
+
# {"input": "العلم نور", "output": "الْعِلْمِ نُورٌ"}
|
|
98
|
+
|
|
99
|
+
cat text.txt | ichkil # read from stdin
|
|
100
|
+
ichkil --model ./model.onnx "محمد" # offline mode
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
## API
|
|
104
|
+
|
|
105
|
+
| Item | Description |
|
|
106
|
+
| --- | --- |
|
|
107
|
+
| `ichkil.diacritize(text: str) -> str` | Vocalize text (module-level convenience, lazy default model). |
|
|
108
|
+
| `ichkil.predict_labels(text: str) -> list[int]` | Label id (0–12) per character. |
|
|
109
|
+
| `Diacritizer` | Constructor: `repo_id`, `revision`, `cache_dir`, `token`, `verify_checksum`, `num_threads`. |
|
|
110
|
+
| `Diacritizer.from_file(model_path, config_path=None)` | Build from local `model.onnx` (+ `config.json` beside it). |
|
|
111
|
+
| `Diacritizer.diacritize(text)` / `.predict_labels(text)` | Instance methods (same behavior). |
|
|
112
|
+
| `ichkil.pipeline` | Pure text helpers: `strip_diacritics`, `encode`, `decode`, `attach`, label tables. |
|
|
113
|
+
| Errors | `IchkilError` ← `ChecksumError`, `ModelLoadError`, `InferenceError`. |
|
|
114
|
+
| `ichkil.reset_default()` | Drop the lazily-created default instance (useful in tests). |
|
|
115
|
+
|
|
116
|
+
### Label contract
|
|
117
|
+
|
|
118
|
+
The model predicts exactly one label per input character:
|
|
119
|
+
|
|
120
|
+
| id | meaning | id | meaning |
|
|
121
|
+
| --- | --- | --- | --- |
|
|
122
|
+
| 0 | non-letter token / padding | 7 | shadda + fatha (ّـَ) |
|
|
123
|
+
| 1 | letter, no diacritic | 8 | shadda + damma (ّـُ) |
|
|
124
|
+
| 2 | fatha (ـَ) | 9 | shadda + kasra (ّـِ) |
|
|
125
|
+
| 3 | damma (ـُ) | 10 | tanwin fatha (ـً) |
|
|
126
|
+
| 4 | kasra (ـِ) | 11 | tanwin damma (ـٌ) |
|
|
127
|
+
| 5 | sukun (ـْ) | 12 | tanwin kasra (ـٍ) |
|
|
128
|
+
| 6 | shadda (ـّ) | | |
|
|
129
|
+
|
|
130
|
+
Already-vocalized input is stripped first, so `diacritize` is idempotent.
|
|
131
|
+
Non-Arabic characters (Latin letters, digits, punctuation, spaces) pass through
|
|
132
|
+
unchanged.
|
|
133
|
+
|
|
134
|
+
## Model provenance
|
|
135
|
+
|
|
136
|
+
- Repository: [`ichkil/ichkil`](https://huggingface.co/ichkil/ichkil)
|
|
137
|
+
(`model.onnx`, `config.json`, `SHA256SUMS`)
|
|
138
|
+
- SHA-256 of `model.onnx`: `9055816b214346b0a8044a4a8deceb36e818dffd0b04699715ea23883bf54744`
|
|
139
|
+
- Contract: `input_ids` int64 `[batch, seq]` → `logits` float32 `[batch, seq, 13]`;
|
|
140
|
+
sequence length fully dynamic (up to `max_seq_len = 1024` in the golden contract)
|
|
141
|
+
- Quality: DER 2.80 % on the reference evaluation set; ONNX output is bit-identical
|
|
142
|
+
to the PyTorch reference (equivalence verified)
|
|
143
|
+
- The cross-runtime golden vectors shipped in `tests/golden.json` are a verbatim
|
|
144
|
+
copy of the core repository's `golden/vectors.json`
|
|
145
|
+
|
|
146
|
+
## Development
|
|
147
|
+
|
|
148
|
+
```bash
|
|
149
|
+
python -m venv .venv && source .venv/bin/activate
|
|
150
|
+
pip install -e ".[dev]"
|
|
151
|
+
pytest # pure pipeline tests (no network)
|
|
152
|
+
pytest -m integration # + model tests (downloads once, ~5 MB)
|
|
153
|
+
ruff check . && ruff format --check . # lint
|
|
154
|
+
```
|
|
155
|
+
|
|
156
|
+
## Publishing
|
|
157
|
+
|
|
158
|
+
`v*` tags are built and published to PyPI via **GitHub trusted publishing (OIDC)** —
|
|
159
|
+
no API token is stored anywhere. One-time setup on
|
|
160
|
+
[pypi.org/project/ichkil](https://pypi.org/project/ichkil/): *Project settings →
|
|
161
|
+
Publishing → Add a publisher* with `github.com/Ichkil/ichkil-python`,
|
|
162
|
+
`workflow: publish.yml`.
|
|
163
|
+
|
|
164
|
+
## Sibling libraries
|
|
165
|
+
|
|
166
|
+
- [ichkil-js](https://github.com/Ichkil/ichkil-js) — npm package (Node.js + browser)
|
|
167
|
+
- [ichkil-go](https://github.com/Ichkil/ichkil-go) — Go module
|
|
168
|
+
- [Core repo](https://github.com/Ichkil/ichkil) — training, evaluation, model spec,
|
|
169
|
+
golden vectors
|
|
170
|
+
|
|
171
|
+
## License
|
|
172
|
+
|
|
173
|
+
[MIT](LICENSE) © 2026 Maaouia BenHamed
|
ichkil-1.0.0/README.md
ADDED
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
# ichkil — Arabic Tashkeel for Python
|
|
2
|
+
|
|
3
|
+
> Vocalize bare Arabic text with a tiny, self-contained ONNX model (≈5 MB, CPU-only,
|
|
4
|
+
> no GPU, no tokenizer, no server). One call, fully offline after the first run.
|
|
5
|
+
|
|
6
|
+
[](https://github.com/Ichkil/ichkil-python/actions/workflows/ci.yml)
|
|
7
|
+
[](https://pypi.org/project/ichkil/)
|
|
8
|
+
[](https://pypi.org/project/ichkil/)
|
|
9
|
+
[](LICENSE)
|
|
10
|
+
[](https://huggingface.co/ichkil/ichkil)
|
|
11
|
+
[](https://huggingface.co/spaces/ichkil/ichkil-demo)
|
|
12
|
+
|
|
13
|
+
`ichkil` downloads its model from [Hugging Face](https://huggingface.co/ichkil/ichkil)
|
|
14
|
+
on first use, verifies its SHA-256, and runs inference with
|
|
15
|
+
[ONNX Runtime](https://onnxruntime.ai/). After that it is 100 % offline.
|
|
16
|
+
|
|
17
|
+
```python
|
|
18
|
+
import ichkil
|
|
19
|
+
|
|
20
|
+
ichkil.diacritize("محمد قرأ الكتاب في المدرسة")
|
|
21
|
+
# -> 'مُحَمَّدٌ قَرَأَ الْكِتَابَ فِي الْمَدْرَسَةِ'
|
|
22
|
+
```
|
|
23
|
+
|
|
24
|
+
## Install
|
|
25
|
+
|
|
26
|
+
```bash
|
|
27
|
+
pip install ichkil
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
Requires Python ≥ 3.9 (tested on 3.9 – 3.12). Dependencies: `huggingface_hub`,
|
|
31
|
+
`onnxruntime` (CPU).
|
|
32
|
+
|
|
33
|
+
## Quickstart
|
|
34
|
+
|
|
35
|
+
```python
|
|
36
|
+
import ichkil
|
|
37
|
+
|
|
38
|
+
# one-liner (lazy model download on first call, then cached)
|
|
39
|
+
ichkil.diacritize("العلم نور والجهل ظلمة")
|
|
40
|
+
# -> 'الْعِلْمِ نُورٌ وَالْجَهْلُ ظُلْمَةُ'
|
|
41
|
+
|
|
42
|
+
# explicit control
|
|
43
|
+
from ichkil import Diacritizer
|
|
44
|
+
|
|
45
|
+
d = Diacritizer() # downloads + verifies + caches the model
|
|
46
|
+
d.diacritize("القهوة جميلة في الصباح")
|
|
47
|
+
# -> 'الْقَهْوَةِ جَمِيلَةٌ فِي الصَّبَاحِ'
|
|
48
|
+
|
|
49
|
+
labels = d.predict_labels("كتاب") # [4, 2, 1, 4] (one label id per character, 0..12)
|
|
50
|
+
|
|
51
|
+
# fully offline: point at local files
|
|
52
|
+
d = Diacritizer.from_file("./model.onnx")
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
### Command line
|
|
56
|
+
|
|
57
|
+
```bash
|
|
58
|
+
pip install ichkil
|
|
59
|
+
ichkil "محمد قرأ الكتاب"
|
|
60
|
+
# مُحَمَّدٌ قَرَأَ الْكِتَابَ
|
|
61
|
+
|
|
62
|
+
ichkil --json "العلم نور"
|
|
63
|
+
# {"input": "العلم نور", "output": "الْعِلْمِ نُورٌ"}
|
|
64
|
+
|
|
65
|
+
cat text.txt | ichkil # read from stdin
|
|
66
|
+
ichkil --model ./model.onnx "محمد" # offline mode
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
## API
|
|
70
|
+
|
|
71
|
+
| Item | Description |
|
|
72
|
+
| --- | --- |
|
|
73
|
+
| `ichkil.diacritize(text: str) -> str` | Vocalize text (module-level convenience, lazy default model). |
|
|
74
|
+
| `ichkil.predict_labels(text: str) -> list[int]` | Label id (0–12) per character. |
|
|
75
|
+
| `Diacritizer` | Constructor: `repo_id`, `revision`, `cache_dir`, `token`, `verify_checksum`, `num_threads`. |
|
|
76
|
+
| `Diacritizer.from_file(model_path, config_path=None)` | Build from local `model.onnx` (+ `config.json` beside it). |
|
|
77
|
+
| `Diacritizer.diacritize(text)` / `.predict_labels(text)` | Instance methods (same behavior). |
|
|
78
|
+
| `ichkil.pipeline` | Pure text helpers: `strip_diacritics`, `encode`, `decode`, `attach`, label tables. |
|
|
79
|
+
| Errors | `IchkilError` ← `ChecksumError`, `ModelLoadError`, `InferenceError`. |
|
|
80
|
+
| `ichkil.reset_default()` | Drop the lazily-created default instance (useful in tests). |
|
|
81
|
+
|
|
82
|
+
### Label contract
|
|
83
|
+
|
|
84
|
+
The model predicts exactly one label per input character:
|
|
85
|
+
|
|
86
|
+
| id | meaning | id | meaning |
|
|
87
|
+
| --- | --- | --- | --- |
|
|
88
|
+
| 0 | non-letter token / padding | 7 | shadda + fatha (ّـَ) |
|
|
89
|
+
| 1 | letter, no diacritic | 8 | shadda + damma (ّـُ) |
|
|
90
|
+
| 2 | fatha (ـَ) | 9 | shadda + kasra (ّـِ) |
|
|
91
|
+
| 3 | damma (ـُ) | 10 | tanwin fatha (ـً) |
|
|
92
|
+
| 4 | kasra (ـِ) | 11 | tanwin damma (ـٌ) |
|
|
93
|
+
| 5 | sukun (ـْ) | 12 | tanwin kasra (ـٍ) |
|
|
94
|
+
| 6 | shadda (ـّ) | | |
|
|
95
|
+
|
|
96
|
+
Already-vocalized input is stripped first, so `diacritize` is idempotent.
|
|
97
|
+
Non-Arabic characters (Latin letters, digits, punctuation, spaces) pass through
|
|
98
|
+
unchanged.
|
|
99
|
+
|
|
100
|
+
## Model provenance
|
|
101
|
+
|
|
102
|
+
- Repository: [`ichkil/ichkil`](https://huggingface.co/ichkil/ichkil)
|
|
103
|
+
(`model.onnx`, `config.json`, `SHA256SUMS`)
|
|
104
|
+
- SHA-256 of `model.onnx`: `9055816b214346b0a8044a4a8deceb36e818dffd0b04699715ea23883bf54744`
|
|
105
|
+
- Contract: `input_ids` int64 `[batch, seq]` → `logits` float32 `[batch, seq, 13]`;
|
|
106
|
+
sequence length fully dynamic (up to `max_seq_len = 1024` in the golden contract)
|
|
107
|
+
- Quality: DER 2.80 % on the reference evaluation set; ONNX output is bit-identical
|
|
108
|
+
to the PyTorch reference (equivalence verified)
|
|
109
|
+
- The cross-runtime golden vectors shipped in `tests/golden.json` are a verbatim
|
|
110
|
+
copy of the core repository's `golden/vectors.json`
|
|
111
|
+
|
|
112
|
+
## Development
|
|
113
|
+
|
|
114
|
+
```bash
|
|
115
|
+
python -m venv .venv && source .venv/bin/activate
|
|
116
|
+
pip install -e ".[dev]"
|
|
117
|
+
pytest # pure pipeline tests (no network)
|
|
118
|
+
pytest -m integration # + model tests (downloads once, ~5 MB)
|
|
119
|
+
ruff check . && ruff format --check . # lint
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
## Publishing
|
|
123
|
+
|
|
124
|
+
`v*` tags are built and published to PyPI via **GitHub trusted publishing (OIDC)** —
|
|
125
|
+
no API token is stored anywhere. One-time setup on
|
|
126
|
+
[pypi.org/project/ichkil](https://pypi.org/project/ichkil/): *Project settings →
|
|
127
|
+
Publishing → Add a publisher* with `github.com/Ichkil/ichkil-python`,
|
|
128
|
+
`workflow: publish.yml`.
|
|
129
|
+
|
|
130
|
+
## Sibling libraries
|
|
131
|
+
|
|
132
|
+
- [ichkil-js](https://github.com/Ichkil/ichkil-js) — npm package (Node.js + browser)
|
|
133
|
+
- [ichkil-go](https://github.com/Ichkil/ichkil-go) — Go module
|
|
134
|
+
- [Core repo](https://github.com/Ichkil/ichkil) — training, evaluation, model spec,
|
|
135
|
+
golden vectors
|
|
136
|
+
|
|
137
|
+
## License
|
|
138
|
+
|
|
139
|
+
[MIT](LICENSE) © 2026 Maaouia BenHamed
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
"""Quickstart example for ichkil (Python).
|
|
2
|
+
|
|
3
|
+
Run:
|
|
4
|
+
python examples/quickstart.py
|
|
5
|
+
|
|
6
|
+
The model is downloaded from Hugging Face on first run (~5 MB) and cached
|
|
7
|
+
under ``~/.cache/huggingface``; afterwards this script runs offline.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import ichkil
|
|
13
|
+
from ichkil import Diacritizer
|
|
14
|
+
|
|
15
|
+
EXAMPLES = [
|
|
16
|
+
"محمد قرأ الكتاب في المدرسة",
|
|
17
|
+
"العلم نور والجهل ظلمة",
|
|
18
|
+
"القهوة جميلة في الصباح",
|
|
19
|
+
"مرحبا، كيف حالك؟",
|
|
20
|
+
"hello world — mixed input",
|
|
21
|
+
]
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def main() -> None:
|
|
25
|
+
print(f"ichkil {ichkil.__version__}\n")
|
|
26
|
+
|
|
27
|
+
for text in EXAMPLES:
|
|
28
|
+
print(f" {text}")
|
|
29
|
+
print(f" -> {ichkil.diacritize(text)}\n")
|
|
30
|
+
|
|
31
|
+
d = Diacritizer()
|
|
32
|
+
print("labels for 'كتاب':", d.predict_labels("كتاب"))
|
|
33
|
+
print("idempotent check:", d.diacritize(d.diacritize("مدرسة")) == d.diacritize("مدرسة"))
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
if __name__ == "__main__":
|
|
37
|
+
main()
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling>=1.21"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "ichkil"
|
|
7
|
+
version = "1.0.0"
|
|
8
|
+
description = "Arabic Tashkeel (diacritization) — a tiny, self-contained ONNX model with a clean Python API"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.9"
|
|
11
|
+
license = { text = "MIT" }
|
|
12
|
+
authors = [{ name = "Maaouia BenHamed" }]
|
|
13
|
+
keywords = [
|
|
14
|
+
"arabic",
|
|
15
|
+
"tashkeel",
|
|
16
|
+
"diacritization",
|
|
17
|
+
"diacritics",
|
|
18
|
+
"harakat",
|
|
19
|
+
"nlp",
|
|
20
|
+
"onnx",
|
|
21
|
+
"onnxruntime",
|
|
22
|
+
"cpu",
|
|
23
|
+
"lightweight",
|
|
24
|
+
]
|
|
25
|
+
classifiers = [
|
|
26
|
+
"Development Status :: 5 - Production/Stable",
|
|
27
|
+
"Intended Audience :: Developers",
|
|
28
|
+
"License :: OSI Approved :: MIT License",
|
|
29
|
+
"Operating System :: OS Independent",
|
|
30
|
+
"Programming Language :: Python :: 3",
|
|
31
|
+
"Programming Language :: Python :: 3.9",
|
|
32
|
+
"Programming Language :: Python :: 3.10",
|
|
33
|
+
"Programming Language :: Python :: 3.11",
|
|
34
|
+
"Programming Language :: Python :: 3.12",
|
|
35
|
+
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
|
36
|
+
"Topic :: Text Processing :: Linguistic",
|
|
37
|
+
]
|
|
38
|
+
dependencies = [
|
|
39
|
+
"huggingface_hub>=0.24",
|
|
40
|
+
"onnxruntime>=1.19,<2",
|
|
41
|
+
]
|
|
42
|
+
|
|
43
|
+
[project.optional-dependencies]
|
|
44
|
+
dev = [
|
|
45
|
+
"build>=1.2",
|
|
46
|
+
"pytest>=7",
|
|
47
|
+
"pytest-cov>=4",
|
|
48
|
+
"ruff>=0.4",
|
|
49
|
+
]
|
|
50
|
+
|
|
51
|
+
[project.scripts]
|
|
52
|
+
ichkil = "ichkil.cli:main"
|
|
53
|
+
|
|
54
|
+
[project.urls]
|
|
55
|
+
Homepage = "https://github.com/Ichkil/ichkil-python"
|
|
56
|
+
Repository = "https://github.com/Ichkil/ichkil-python"
|
|
57
|
+
Issues = "https://github.com/Ichkil/ichkil-python/issues"
|
|
58
|
+
"Hugging Face" = "https://huggingface.co/ichkil/ichkil"
|
|
59
|
+
"Live demo (browser)" = "https://huggingface.co/spaces/ichkil/ichkil-demo"
|
|
60
|
+
|
|
61
|
+
[tool.hatch.build.targets.wheel]
|
|
62
|
+
packages = ["src/ichkil"]
|
|
63
|
+
|
|
64
|
+
[tool.pytest.ini_options]
|
|
65
|
+
testpaths = ["tests"]
|
|
66
|
+
addopts = "-q"
|
|
67
|
+
markers = [
|
|
68
|
+
"integration: downloads the model from Hugging Face on first use (deselect with -m 'not integration')",
|
|
69
|
+
]
|
|
70
|
+
|
|
71
|
+
[tool.ruff]
|
|
72
|
+
line-length = 100
|
|
73
|
+
target-version = "py39"
|
|
74
|
+
|
|
75
|
+
[tool.ruff.lint]
|
|
76
|
+
select = ["E", "F", "W", "I", "UP", "B", "SIM", "C4"]
|
|
77
|
+
|
|
78
|
+
[tool.coverage.run]
|
|
79
|
+
source = ["ichkil"]
|