misar 1.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- misar-1.0.0/LICENSE +21 -0
- misar-1.0.0/PKG-INFO +281 -0
- misar-1.0.0/README.md +244 -0
- misar-1.0.0/pyproject.toml +62 -0
- misar-1.0.0/setup.cfg +4 -0
- misar-1.0.0/src/misar/__init__.py +36 -0
- misar-1.0.0/src/misar/__main__.py +5 -0
- misar-1.0.0/src/misar/benchmark.py +119 -0
- misar-1.0.0/src/misar/cli.py +124 -0
- misar-1.0.0/src/misar/lexicons.py +125 -0
- misar-1.0.0/src/misar/pipelines.py +191 -0
- misar-1.0.0/src/misar/profiler.py +171 -0
- misar-1.0.0/src/misar/router.py +237 -0
- misar-1.0.0/src/misar/tokenizer.py +305 -0
- misar-1.0.0/src/misar/web/__init__.py +1 -0
- misar-1.0.0/src/misar/web/index.html +1344 -0
- misar-1.0.0/src/misar/web/server.py +289 -0
- misar-1.0.0/src/misar.egg-info/PKG-INFO +281 -0
- misar-1.0.0/src/misar.egg-info/SOURCES.txt +22 -0
- misar-1.0.0/src/misar.egg-info/dependency_links.txt +1 -0
- misar-1.0.0/src/misar.egg-info/entry_points.txt +2 -0
- misar-1.0.0/src/misar.egg-info/requires.txt +9 -0
- misar-1.0.0/src/misar.egg-info/top_level.txt +1 -0
- misar-1.0.0/tests/test_misar.py +289 -0
misar-1.0.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Muneer Jarab Matrak Al-Theeb — Misar NLP
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
misar-1.0.0/PKG-INFO
ADDED
|
@@ -0,0 +1,281 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: misar
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: Misar (مِسار): Adaptive context-aware Arabic NLP preprocessing router for optimizing Transformer language models.
|
|
5
|
+
Author: Muneer Jarab Matrak Al-Theeb
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/RealMtrx/misar
|
|
8
|
+
Project-URL: Repository, https://github.com/RealMtrx/misar
|
|
9
|
+
Project-URL: Issues, https://github.com/RealMtrx/misar/issues
|
|
10
|
+
Keywords: arabic,nlp,preprocessing,router,arabert,camelbert,llm,tokenization,dialect,saudi-arabia
|
|
11
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: Intended Audience :: Science/Research
|
|
14
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
15
|
+
Classifier: Natural Language :: Arabic
|
|
16
|
+
Classifier: Operating System :: OS Independent
|
|
17
|
+
Classifier: Programming Language :: Python :: 3
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
23
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
24
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
25
|
+
Classifier: Topic :: Text Processing :: Linguistic
|
|
26
|
+
Requires-Python: >=3.9
|
|
27
|
+
Description-Content-Type: text/markdown
|
|
28
|
+
License-File: LICENSE
|
|
29
|
+
Provides-Extra: huggingface
|
|
30
|
+
Requires-Dist: transformers>=4.35.0; extra == "huggingface"
|
|
31
|
+
Requires-Dist: torch>=2.0.0; extra == "huggingface"
|
|
32
|
+
Provides-Extra: dev
|
|
33
|
+
Requires-Dist: pytest>=7.4.0; extra == "dev"
|
|
34
|
+
Requires-Dist: build>=1.0.0; extra == "dev"
|
|
35
|
+
Requires-Dist: twine>=5.0.0; extra == "dev"
|
|
36
|
+
Dynamic: license-file
|
|
37
|
+
|
|
38
|
+
<div align="center">
|
|
39
|
+
|
|
40
|
+
# مِسار — Misar
|
|
41
|
+
|
|
42
|
+
**Adaptive $O(n)$ Context-Aware Arabic NLP Preprocessing Router for Transformer Language Models**
|
|
43
|
+
**الموجّه اللغوي التكيفي لتعزيز كفاءة ودقة معالجة اللغة العربية في نماذج الذكاء الاصطناعي**
|
|
44
|
+
|
|
45
|
+
[](https://github.com/RealMtrx/misar/actions/workflows/ci.yml)
|
|
46
|
+
[](https://www.python.org/downloads/)
|
|
47
|
+
[](#installation)
|
|
48
|
+
[](https://github.com/RealMtrx/misar/blob/main/LICENSE)
|
|
49
|
+
[](#empirical-benchmarks)
|
|
50
|
+
[](#empirical-benchmarks)
|
|
51
|
+
|
|
52
|
+
[Architecture](#architecture) • [Benchmarks](#empirical-benchmarks) • [Installation](#installation) • [Quick Start](#quick-start) • [CLI & Workbench](#cli--web-workbench) • [Citation](#citation) • [النسخة العربية](#-نظرة-عامة-بالعربية)
|
|
53
|
+
|
|
54
|
+
<br/>
|
|
55
|
+
|
|
56
|
+
<img src="https://raw.githubusercontent.com/RealMtrx/misar/main/docs/misar-architecture.png" alt="Misar Adaptive Arabic NLP Preprocessing Router Architecture" width="100%" />
|
|
57
|
+
|
|
58
|
+
</div>
|
|
59
|
+
|
|
60
|
+
---
|
|
61
|
+
|
|
62
|
+
## Why Misar?
|
|
63
|
+
|
|
64
|
+
Modern Arabic NLP deployments face a structural bottleneck: forcing heterogeneous Arabic text (**Modern Standard Arabic**, **classical diacritized prose**, **Gulf/Egyptian/Levantine/Maghrebi dialects**, and **noisy social media**) through a single static preprocessing pipeline actively degrades both accuracy and inference economics:
|
|
65
|
+
|
|
66
|
+
1. **Legacy Normalization Damages Modern Transformers:** Classical regex pipelines blindly collapse `ة → ه` and `أ/إ/آ → ا`. However, modern Transformer vocabularies such as **AraBERTv2** (`64k` WordPiece) and **CAMeLBERT** explicitly preserve Hamzas and Taa Marbuta—meaning legacy normalization shatters known whole words (e.g., `الحوسبة` as `1` token splits into `الحو` + `##سبه` as `2` tokens).
|
|
67
|
+
2. **Dialectal Sentiment & Negation Erasure:** Over-aggressive cleaning distorts critical colloquial negation and sentiment markers (`مو`, `مب`, `مافيه`, `ماكو`), reversing downstream classification polarity.
|
|
68
|
+
3. **Diacritic, Tatweel & Elongation Token Explosion:** Unstripped harakat (`تشكيل`), kashida (`ـ`), and character stretches (`بطييييء`, `السيرفررررر`) trigger OOV `[UNK]` blindness or inflate subword token counts—multiplying Transformer $O(L^2)$ self-attention cost and API billing.
|
|
69
|
+
|
|
70
|
+
**Misar (`مِسار`)** solves all three challenges **without retraining any model (`Zero Parameter Update`)** by acting as an ultra-lightweight, zero-dependency, linear-time $O(n)$ routing gateway directly in front of your Transformer or LLM pipeline.
|
|
71
|
+
|
|
72
|
+
---
|
|
73
|
+
|
|
74
|
+
## Architecture
|
|
75
|
+
|
|
76
|
+
Every incoming text passes once through `LinguisticProfiler` (`< 0.25 ms` on CPU) to compute five quantitative signals (`dialect_ratio` $P_{\text{dial}}$, `diacritic_density` $D_{\text{diac}}$, `elongation_noise_score` $E_{\text{noise}}$, character Shannon Entropy $H_{\text{char}}$, and `code_switching_ratio` $R_{\text{latin}}$), then dispatches dynamically to the optimal micro-pipeline:
|
|
77
|
+
|
|
78
|
+
```mermaid
|
|
79
|
+
flowchart LR
|
|
80
|
+
A["Raw Arabic Input<br/>(MSA / Diacritized / Dialect / Social)"] --> B["LinguisticProfiler O(n)<br/>P_dial · D_diac · E_noise · H_char · R_latin"]
|
|
81
|
+
B -->|"High Noise / Tatweel / Stretches ≥ 3"| C["NOISY_SOCIAL (+ Hybrid Dialect)<br/>Gemination-Safe Collapse + Tag Unpacking"]
|
|
82
|
+
B -->|"Dialect Markers / Protected Negations"| D["DIALECT_PRESERVE<br/>Negation Protection + Vocab-Aligned Cleanup"]
|
|
83
|
+
B -->|"Clean MSA / Classical Diacritized"| E["FAST_MSA<br/>Strip Harakat · Preserve أ/إ/آ, ة, ى"]
|
|
84
|
+
C --> F["AraBERTv2 / CAMeLBERT / ALLaM / LLMs<br/>(-46.57% Tokens · 0 [UNK])"]
|
|
85
|
+
D --> F
|
|
86
|
+
E --> F
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
---
|
|
90
|
+
|
|
91
|
+
## Empirical Benchmarks
|
|
92
|
+
|
|
93
|
+
Measured reproducibly via `python benchmarks/run_benchmark.py` against the authentic **64,000-token `aubmindlab/bert-base-arabertv02` WordPiece vocabulary** across 15 curated multi-domain samples ([`latest_benchmark_report.json`](https://github.com/RealMtrx/misar/blob/main/benchmarks/latest_benchmark_report.json)):
|
|
94
|
+
|
|
95
|
+
| Domain Category | Samples | Raw Tokens | Legacy Static (`ة→ه`, `أ→ا`) | **Misar Adaptive** | Misar Gain vs. Raw |
|
|
96
|
+
| :--- | :---: | :---: | :---: | :---: | :---: |
|
|
97
|
+
| **MSA Engineering & Code-Switch** | 3 | 57 | 79 *(+38.6% fragmentation)* | **57** | Preserves `100%` orthography (`-27.85%` vs Legacy) |
|
|
98
|
+
| **Classical Diacritized (Heritage)** | 3 | 186 *(147 OOV `[UNK]`)* | 48 | **39** | **`-79.03%`** tokens & **`100%`** word recovery (`0 [UNK]`) |
|
|
99
|
+
| **Saudi / Gulf / Levantine / Maghrebi Dev** | 6 | 121 | 131 | **117** | **`-3.31%`** tokens (`-10.69%` vs Legacy) + negations intact |
|
|
100
|
+
| **Noisy Tech Social & Code-Switching** | 3 | 102 | 80 | **36** | **`-64.71%`** tokens + gemination preserved |
|
|
101
|
+
| **Full Corpus Summary** | **15** | **466** | **338** | **249** | **`-46.57%` vs Raw · `-26.33%` vs Legacy · `0.223 ms/text`** |
|
|
102
|
+
|
|
103
|
+
> **Key Scientific Proof:** On Standard Arabic technical prose (`MSA Tech / Engineering`), blind legacy normalization inflated token count from **57 → 79** due to `ة → ه` subword splitting, while Misar maintained **57** intact vocabulary tokens (`TFR ≈ 1.0`). Across all 15 multi-domain inputs, Misar saved **46.57%** of total tokens vs. raw text and **26.33%** vs. legacy normalization (recovering **156 → 0 `[UNK]`** tokens) with **0.223 ms** mean latency on pure CPU.
|
|
104
|
+
|
|
105
|
+
---
|
|
106
|
+
|
|
107
|
+
## Installation
|
|
108
|
+
|
|
109
|
+
Core `misar` requires **zero mandatory external packages** (Python `3.9+` Standard Library only):
|
|
110
|
+
|
|
111
|
+
```bash
|
|
112
|
+
# Install directly from GitHub
|
|
113
|
+
pip install git+https://github.com/RealMtrx/misar.git
|
|
114
|
+
|
|
115
|
+
# Or from PyPI once published
|
|
116
|
+
pip install misar
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
Optional extras for live Hugging Face Transformers wrapping and package development:
|
|
120
|
+
|
|
121
|
+
```bash
|
|
122
|
+
pip install "misar[huggingface]" # transformers + torch
|
|
123
|
+
pip install "misar[dev]" # pytest + build + twine
|
|
124
|
+
```
|
|
125
|
+
|
|
126
|
+
*(Note: On first use of the built-in `WordPieceTokenizer`, Misar caches the official 64k vocabulary file locally under `~/.cache/misar/`. Pass `allow_vocab_download=False` or `--offline` to run completely air-gapped using `HeuristicSubwordTokenizer`.)*
|
|
127
|
+
|
|
128
|
+
---
|
|
129
|
+
|
|
130
|
+
## Quick Start
|
|
131
|
+
|
|
132
|
+
### 1. Dynamic Text Routing & Token Telemetry
|
|
133
|
+
|
|
134
|
+
```python
|
|
135
|
+
from misar import MisarRouter
|
|
136
|
+
|
|
137
|
+
router = MisarRouter() # Uses real AraBERTv2 64k WordPiece vocabulary
|
|
138
|
+
|
|
139
|
+
result = router.route("السيرفررررر بطييييء مررررة ومافيه استجابة في API أبدددداً 😡😡👎 #هندسة_البرمجيات")
|
|
140
|
+
|
|
141
|
+
print("Route: ", result.route_name) # NOISY_SOCIAL
|
|
142
|
+
print("Processed: ", result.processed_text) # السيرفر بطيء مرة ومافيه استجابة في API أبدا هندسة البرمجيات
|
|
143
|
+
print("Tokens: ", result.raw_token_count, "->", result.processed_token_count) # 28 -> 14
|
|
144
|
+
print("Reduction: ", f"-{result.token_reduction_pct:.1f}%") # -50.0%
|
|
145
|
+
print("Latency: ", f"{result.latency_ms:.3f} ms")
|
|
146
|
+
```
|
|
147
|
+
|
|
148
|
+
### 2. Zero-Latency Drop-In String Preprocessing & Batching
|
|
149
|
+
|
|
150
|
+
```python
|
|
151
|
+
from misar import MisarRouter, preprocess
|
|
152
|
+
|
|
153
|
+
# One-line module helper
|
|
154
|
+
clean = preprocess("الســــلام علــــيكم ومافيه مشكلة #تطوير_البرمجيات")
|
|
155
|
+
# -> "السلام عليكم ومافيه مشكلة تطوير البرمجيات"
|
|
156
|
+
|
|
157
|
+
# Fast batch routing for production ETL pipelines
|
|
158
|
+
router = MisarRouter()
|
|
159
|
+
results = router.route_batch([
|
|
160
|
+
"أصدرت عدة دول عربية بياناً مشتركاً لأول مرة عن صافي الأرباح", # -> FAST_MSA (zero dialect false positives)
|
|
161
|
+
"والللله الكود مو معقد أبداً والأخطاء متكررررة", # -> NOISY_SOCIAL (preserves والله and مو)
|
|
162
|
+
])
|
|
163
|
+
```
|
|
164
|
+
|
|
165
|
+
### 3. One-Line Hugging Face Transformers Integration
|
|
166
|
+
|
|
167
|
+
```python
|
|
168
|
+
from transformers import pipeline
|
|
169
|
+
from misar import MisarRouter
|
|
170
|
+
|
|
171
|
+
sentiment = pipeline("sentiment-analysis", model="CAMeL-Lab/bert-base-arabic-camelbert-mix-sentiment")
|
|
172
|
+
smart_sentiment = MisarRouter(tokenizer=sentiment.tokenizer).wrap_hf_pipeline(sentiment)
|
|
173
|
+
|
|
174
|
+
print(smart_sentiment("الكود مو معقد أبداً بس السيرفررررر بطييييء #مشكلة"))
|
|
175
|
+
```
|
|
176
|
+
|
|
177
|
+
---
|
|
178
|
+
|
|
179
|
+
## CLI & Web Workbench
|
|
180
|
+
|
|
181
|
+
Misar includes a clean command-line interface and a self-hosted HTTP server + Arabic RTL Interactive Workbench:
|
|
182
|
+
|
|
183
|
+
```bash
|
|
184
|
+
# Run 6 built-in cross-dialect samples with live AraBERTv2 tokenization
|
|
185
|
+
misar demo
|
|
186
|
+
|
|
187
|
+
# Route a single sentence and inspect JSON telemetry
|
|
188
|
+
misar route "أعلنت منصة المطورين عن ترقية واجهة البرمجة API لدعم نماذج المحولات" --json
|
|
189
|
+
|
|
190
|
+
# Run the reproducible 15-sample benchmark suite
|
|
191
|
+
misar bench
|
|
192
|
+
|
|
193
|
+
# Launch the interactive RTL Web Workbench & hardened REST API on http://127.0.0.1:8050
|
|
194
|
+
misar serve
|
|
195
|
+
```
|
|
196
|
+
|
|
197
|
+
### Embedded REST API Endpoints (`misar serve`)
|
|
198
|
+
|
|
199
|
+
| Method | Endpoint | Description |
|
|
200
|
+
| :--- | :--- | :--- |
|
|
201
|
+
| `GET` | `/api/health` | Service status, version, and active tokenizer backend |
|
|
202
|
+
| `GET` | `/api/suite` | Evaluates the 6 standard cross-domain reference samples |
|
|
203
|
+
| `POST` | `/api/route` | Body `{"text": "..."}` → Returns full `RoutingResult` telemetry |
|
|
204
|
+
| `POST` | `/api/batch` | Body `{"texts": ["...", "..."]}` → Batch routes up to 500 strings (`1 MiB` payload limit) |
|
|
205
|
+
|
|
206
|
+
---
|
|
207
|
+
|
|
208
|
+
## Repository Structure
|
|
209
|
+
|
|
210
|
+
```text
|
|
211
|
+
misar/
|
|
212
|
+
├── .github/workflows/ci.yml # Cross-OS Python 3.9–3.13 CI pipeline
|
|
213
|
+
├── docs/misar-architecture.png # High-resolution system architecture diagram
|
|
214
|
+
├── benchmarks/
|
|
215
|
+
│ ├── run_benchmark.py # Reproducible benchmark runner
|
|
216
|
+
│ └── latest_benchmark_report.json # Verified telemetry snapshot
|
|
217
|
+
├── src/misar/
|
|
218
|
+
│ ├── __init__.py # Public API exports
|
|
219
|
+
│ ├── profiler.py # O(n) 5-signal LinguisticProfiler & Shannon entropy
|
|
220
|
+
│ ├── lexicons.py # MSA-safe multi-region dialect & negation lexicons
|
|
221
|
+
│ ├── pipelines.py # FAST_MSA, DIALECT_PRESERVE & NOISY_SOCIAL passes
|
|
222
|
+
│ ├── tokenizer.py # Pure-Python WordPieceTokenizer + HF & offline adapters
|
|
223
|
+
│ ├── router.py # MisarRouter orchestration & wrap_hf_pipeline
|
|
224
|
+
│ ├── benchmark.py # Shared benchmark suite module
|
|
225
|
+
│ ├── cli.py # `misar` CLI (`demo`, `route`, `bench`, `serve`)
|
|
226
|
+
│ └── web/
|
|
227
|
+
│ ├── server.py # Hardened zero-dependency ThreadingHTTPServer REST API
|
|
228
|
+
│ └── index.html # Interactive Arabic RTL Workbench & token visualizer
|
|
229
|
+
├── tests/test_misar.py # 23 unit, security, server & real-vocab tests
|
|
230
|
+
├── CITATION.cff # Native GitHub repository citation metadata
|
|
231
|
+
├── CONTRIBUTING.md # Developer setup & contribution standards
|
|
232
|
+
├── SECURITY.md # Built-in API & frontend hardening controls
|
|
233
|
+
├── CHANGELOG.md # Formal release history
|
|
234
|
+
├── LICENSE # MIT License
|
|
235
|
+
└── pyproject.toml # PEP 621 PyPI package specification
|
|
236
|
+
```
|
|
237
|
+
|
|
238
|
+
---
|
|
239
|
+
|
|
240
|
+
## Testing & Verification
|
|
241
|
+
|
|
242
|
+
```bash
|
|
243
|
+
python -m unittest discover -s tests -v
|
|
244
|
+
```
|
|
245
|
+
|
|
246
|
+
All **23 automated tests** verify profiler mathematics, MSA homograph false-positive resistance, Saudi/Egyptian/Levantine/Maghrebi negation preservation, gemination protection (`والللله → والله`), markdown stripping, tatweel/diacritic splitting, authentic `AraBERTv2` WordPiece reduction, and REST server HTTP security bounds.
|
|
247
|
+
|
|
248
|
+
---
|
|
249
|
+
|
|
250
|
+
## Citation
|
|
251
|
+
|
|
252
|
+
If you use **Misar** in academic research, benchmarks, or commercial NLP pipelines, please cite:
|
|
253
|
+
|
|
254
|
+
```bibtex
|
|
255
|
+
@software{altheeb2026misar,
|
|
256
|
+
author = {Muneer Jarab Matrak Al-Theeb},
|
|
257
|
+
title = {Misar (مِسار): Adaptive Context-Aware Arabic NLP Preprocessing Router for Transformer Language Models},
|
|
258
|
+
year = {2026},
|
|
259
|
+
version = {1.0.0},
|
|
260
|
+
url = {https://github.com/RealMtrx/misar},
|
|
261
|
+
license = {MIT}
|
|
262
|
+
}
|
|
263
|
+
```
|
|
264
|
+
|
|
265
|
+
---
|
|
266
|
+
|
|
267
|
+
## 📖 نظرة عامة بالعربية
|
|
268
|
+
|
|
269
|
+
**مِسار (`Misar`)** هي مكتبة وموجّه بايثون مفتوح المصدر وخفيف الوزن (بلا أي تبعيات خارجية إجبارية) صُمِّم ليعمل كطبقة بوابة ذكية في زمن خطي $O(n)$ (**~0.22 مللي ثانية** للنص) قبل نماذج المحولات اللغوية العربية مثل `AraBERTv2` و `CAMeLBERT` والنماذج التوليدية (`ALLaM` و `LLMs`) **دون الحاجة إلى إعادة تدريب النموذج (`Zero Parameter Update`)**.
|
|
270
|
+
|
|
271
|
+
* **توجيه ثلاثي المسارات:** يفحص النص في تمريرة واحدة ويختار بين مسار الفصحى الحافظ للإملاء (`FAST_MSA`)، أو مسار حفظ اللهجة وأدوات النفي (`DIALECT_PRESERVE`) لحماية كلمات مثل `مو، مب، مافيه، ماكو`، أو مسار تنقية ضجيج شبكات التواصل (`NOISY_SOCIAL`) الذي يضغط المدود والتطويل مع الحفاظ على الكلمات المشددة (`والللله ← والله`).
|
|
272
|
+
* **توفير التوكنات وتكاليف الـ APIs:** أظهرت التجارب المرجعية الموثقة على معجم `AraBERTv2` خفض إجمالي التوكنات بنسبة **46.57%** مقارنة بالنص الخام و**26.33%** مقارنة بمناهج التطبيع القديمة، مع استعادة **100%** من الكلمات المتأثرة بالتشكيل والكشيدة (`156 → 0 [UNK]`).
|
|
273
|
+
|
|
274
|
+
---
|
|
275
|
+
|
|
276
|
+
<div align="center">
|
|
277
|
+
|
|
278
|
+
Authored and maintained by **Muneer Jarab Matrak Al-Theeb (منير جراب مترك الذيب)**
|
|
279
|
+
Released under the [MIT License](https://github.com/RealMtrx/misar/blob/main/LICENSE) © 2026
|
|
280
|
+
|
|
281
|
+
</div>
|
misar-1.0.0/README.md
ADDED
|
@@ -0,0 +1,244 @@
|
|
|
1
|
+
<div align="center">
|
|
2
|
+
|
|
3
|
+
# مِسار — Misar
|
|
4
|
+
|
|
5
|
+
**Adaptive $O(n)$ Context-Aware Arabic NLP Preprocessing Router for Transformer Language Models**
|
|
6
|
+
**الموجّه اللغوي التكيفي لتعزيز كفاءة ودقة معالجة اللغة العربية في نماذج الذكاء الاصطناعي**
|
|
7
|
+
|
|
8
|
+
[](https://github.com/RealMtrx/misar/actions/workflows/ci.yml)
|
|
9
|
+
[](https://www.python.org/downloads/)
|
|
10
|
+
[](#installation)
|
|
11
|
+
[](https://github.com/RealMtrx/misar/blob/main/LICENSE)
|
|
12
|
+
[](#empirical-benchmarks)
|
|
13
|
+
[](#empirical-benchmarks)
|
|
14
|
+
|
|
15
|
+
[Architecture](#architecture) • [Benchmarks](#empirical-benchmarks) • [Installation](#installation) • [Quick Start](#quick-start) • [CLI & Workbench](#cli--web-workbench) • [Citation](#citation) • [النسخة العربية](#-نظرة-عامة-بالعربية)
|
|
16
|
+
|
|
17
|
+
<br/>
|
|
18
|
+
|
|
19
|
+
<img src="https://raw.githubusercontent.com/RealMtrx/misar/main/docs/misar-architecture.png" alt="Misar Adaptive Arabic NLP Preprocessing Router Architecture" width="100%" />
|
|
20
|
+
|
|
21
|
+
</div>
|
|
22
|
+
|
|
23
|
+
---
|
|
24
|
+
|
|
25
|
+
## Why Misar?
|
|
26
|
+
|
|
27
|
+
Modern Arabic NLP deployments face a structural bottleneck: forcing heterogeneous Arabic text (**Modern Standard Arabic**, **classical diacritized prose**, **Gulf/Egyptian/Levantine/Maghrebi dialects**, and **noisy social media**) through a single static preprocessing pipeline actively degrades both accuracy and inference economics:
|
|
28
|
+
|
|
29
|
+
1. **Legacy Normalization Damages Modern Transformers:** Classical regex pipelines blindly collapse `ة → ه` and `أ/إ/آ → ا`. However, modern Transformer vocabularies such as **AraBERTv2** (`64k` WordPiece) and **CAMeLBERT** explicitly preserve Hamzas and Taa Marbuta—meaning legacy normalization shatters known whole words (e.g., `الحوسبة` as `1` token splits into `الحو` + `##سبه` as `2` tokens).
|
|
30
|
+
2. **Dialectal Sentiment & Negation Erasure:** Over-aggressive cleaning distorts critical colloquial negation and sentiment markers (`مو`, `مب`, `مافيه`, `ماكو`), reversing downstream classification polarity.
|
|
31
|
+
3. **Diacritic, Tatweel & Elongation Token Explosion:** Unstripped harakat (`تشكيل`), kashida (`ـ`), and character stretches (`بطييييء`, `السيرفررررر`) trigger OOV `[UNK]` blindness or inflate subword token counts—multiplying Transformer $O(L^2)$ self-attention cost and API billing.
|
|
32
|
+
|
|
33
|
+
**Misar (`مِسار`)** solves all three challenges **without retraining any model (`Zero Parameter Update`)** by acting as an ultra-lightweight, zero-dependency, linear-time $O(n)$ routing gateway directly in front of your Transformer or LLM pipeline.
|
|
34
|
+
|
|
35
|
+
---
|
|
36
|
+
|
|
37
|
+
## Architecture
|
|
38
|
+
|
|
39
|
+
Every incoming text passes once through `LinguisticProfiler` (`< 0.25 ms` on CPU) to compute five quantitative signals (`dialect_ratio` $P_{\text{dial}}$, `diacritic_density` $D_{\text{diac}}$, `elongation_noise_score` $E_{\text{noise}}$, character Shannon Entropy $H_{\text{char}}$, and `code_switching_ratio` $R_{\text{latin}}$), then dispatches dynamically to the optimal micro-pipeline:
|
|
40
|
+
|
|
41
|
+
```mermaid
|
|
42
|
+
flowchart LR
|
|
43
|
+
A["Raw Arabic Input<br/>(MSA / Diacritized / Dialect / Social)"] --> B["LinguisticProfiler O(n)<br/>P_dial · D_diac · E_noise · H_char · R_latin"]
|
|
44
|
+
B -->|"High Noise / Tatweel / Stretches ≥ 3"| C["NOISY_SOCIAL (+ Hybrid Dialect)<br/>Gemination-Safe Collapse + Tag Unpacking"]
|
|
45
|
+
B -->|"Dialect Markers / Protected Negations"| D["DIALECT_PRESERVE<br/>Negation Protection + Vocab-Aligned Cleanup"]
|
|
46
|
+
B -->|"Clean MSA / Classical Diacritized"| E["FAST_MSA<br/>Strip Harakat · Preserve أ/إ/آ, ة, ى"]
|
|
47
|
+
C --> F["AraBERTv2 / CAMeLBERT / ALLaM / LLMs<br/>(-46.57% Tokens · 0 [UNK])"]
|
|
48
|
+
D --> F
|
|
49
|
+
E --> F
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
---
|
|
53
|
+
|
|
54
|
+
## Empirical Benchmarks
|
|
55
|
+
|
|
56
|
+
Measured reproducibly via `python benchmarks/run_benchmark.py` against the authentic **64,000-token `aubmindlab/bert-base-arabertv02` WordPiece vocabulary** across 15 curated multi-domain samples ([`latest_benchmark_report.json`](https://github.com/RealMtrx/misar/blob/main/benchmarks/latest_benchmark_report.json)):
|
|
57
|
+
|
|
58
|
+
| Domain Category | Samples | Raw Tokens | Legacy Static (`ة→ه`, `أ→ا`) | **Misar Adaptive** | Misar Gain vs. Raw |
|
|
59
|
+
| :--- | :---: | :---: | :---: | :---: | :---: |
|
|
60
|
+
| **MSA Engineering & Code-Switch** | 3 | 57 | 79 *(+38.6% fragmentation)* | **57** | Preserves `100%` orthography (`-27.85%` vs Legacy) |
|
|
61
|
+
| **Classical Diacritized (Heritage)** | 3 | 186 *(147 OOV `[UNK]`)* | 48 | **39** | **`-79.03%`** tokens & **`100%`** word recovery (`0 [UNK]`) |
|
|
62
|
+
| **Saudi / Gulf / Levantine / Maghrebi Dev** | 6 | 121 | 131 | **117** | **`-3.31%`** tokens (`-10.69%` vs Legacy) + negations intact |
|
|
63
|
+
| **Noisy Tech Social & Code-Switching** | 3 | 102 | 80 | **36** | **`-64.71%`** tokens + gemination preserved |
|
|
64
|
+
| **Full Corpus Summary** | **15** | **466** | **338** | **249** | **`-46.57%` vs Raw · `-26.33%` vs Legacy · `0.223 ms/text`** |
|
|
65
|
+
|
|
66
|
+
> **Key Scientific Proof:** On Standard Arabic technical prose (`MSA Tech / Engineering`), blind legacy normalization inflated token count from **57 → 79** due to `ة → ه` subword splitting, while Misar maintained **57** intact vocabulary tokens (`TFR ≈ 1.0`). Across all 15 multi-domain inputs, Misar saved **46.57%** of total tokens vs. raw text and **26.33%** vs. legacy normalization (recovering **156 → 0 `[UNK]`** tokens) with **0.223 ms** mean latency on pure CPU.
|
|
67
|
+
|
|
68
|
+
---
|
|
69
|
+
|
|
70
|
+
## Installation
|
|
71
|
+
|
|
72
|
+
Core `misar` requires **zero mandatory external packages** (Python `3.9+` Standard Library only):
|
|
73
|
+
|
|
74
|
+
```bash
|
|
75
|
+
# Install directly from GitHub
|
|
76
|
+
pip install git+https://github.com/RealMtrx/misar.git
|
|
77
|
+
|
|
78
|
+
# Or from PyPI once published
|
|
79
|
+
pip install misar
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
Optional extras for live Hugging Face Transformers wrapping and package development:
|
|
83
|
+
|
|
84
|
+
```bash
|
|
85
|
+
pip install "misar[huggingface]" # transformers + torch
|
|
86
|
+
pip install "misar[dev]" # pytest + build + twine
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
*(Note: On first use of the built-in `WordPieceTokenizer`, Misar caches the official 64k vocabulary file locally under `~/.cache/misar/`. Pass `allow_vocab_download=False` or `--offline` to run completely air-gapped using `HeuristicSubwordTokenizer`.)*
|
|
90
|
+
|
|
91
|
+
---
|
|
92
|
+
|
|
93
|
+
## Quick Start
|
|
94
|
+
|
|
95
|
+
### 1. Dynamic Text Routing & Token Telemetry
|
|
96
|
+
|
|
97
|
+
```python
|
|
98
|
+
from misar import MisarRouter
|
|
99
|
+
|
|
100
|
+
router = MisarRouter() # Uses real AraBERTv2 64k WordPiece vocabulary
|
|
101
|
+
|
|
102
|
+
result = router.route("السيرفررررر بطييييء مررررة ومافيه استجابة في API أبدددداً 😡😡👎 #هندسة_البرمجيات")
|
|
103
|
+
|
|
104
|
+
print("Route: ", result.route_name) # NOISY_SOCIAL
|
|
105
|
+
print("Processed: ", result.processed_text) # السيرفر بطيء مرة ومافيه استجابة في API أبدا هندسة البرمجيات
|
|
106
|
+
print("Tokens: ", result.raw_token_count, "->", result.processed_token_count) # 28 -> 14
|
|
107
|
+
print("Reduction: ", f"-{result.token_reduction_pct:.1f}%") # -50.0%
|
|
108
|
+
print("Latency: ", f"{result.latency_ms:.3f} ms")
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
### 2. Zero-Latency Drop-In String Preprocessing & Batching
|
|
112
|
+
|
|
113
|
+
```python
|
|
114
|
+
from misar import MisarRouter, preprocess
|
|
115
|
+
|
|
116
|
+
# One-line module helper
|
|
117
|
+
clean = preprocess("الســــلام علــــيكم ومافيه مشكلة #تطوير_البرمجيات")
|
|
118
|
+
# -> "السلام عليكم ومافيه مشكلة تطوير البرمجيات"
|
|
119
|
+
|
|
120
|
+
# Fast batch routing for production ETL pipelines
|
|
121
|
+
router = MisarRouter()
|
|
122
|
+
results = router.route_batch([
|
|
123
|
+
"أصدرت عدة دول عربية بياناً مشتركاً لأول مرة عن صافي الأرباح", # -> FAST_MSA (zero dialect false positives)
|
|
124
|
+
"والللله الكود مو معقد أبداً والأخطاء متكررررة", # -> NOISY_SOCIAL (preserves والله and مو)
|
|
125
|
+
])
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
### 3. One-Line Hugging Face Transformers Integration
|
|
129
|
+
|
|
130
|
+
```python
|
|
131
|
+
from transformers import pipeline
|
|
132
|
+
from misar import MisarRouter
|
|
133
|
+
|
|
134
|
+
sentiment = pipeline("sentiment-analysis", model="CAMeL-Lab/bert-base-arabic-camelbert-mix-sentiment")
|
|
135
|
+
smart_sentiment = MisarRouter(tokenizer=sentiment.tokenizer).wrap_hf_pipeline(sentiment)
|
|
136
|
+
|
|
137
|
+
print(smart_sentiment("الكود مو معقد أبداً بس السيرفررررر بطييييء #مشكلة"))
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
---
|
|
141
|
+
|
|
142
|
+
## CLI & Web Workbench
|
|
143
|
+
|
|
144
|
+
Misar includes a clean command-line interface and a self-hosted HTTP server + Arabic RTL Interactive Workbench:
|
|
145
|
+
|
|
146
|
+
```bash
|
|
147
|
+
# Run 6 built-in cross-dialect samples with live AraBERTv2 tokenization
|
|
148
|
+
misar demo
|
|
149
|
+
|
|
150
|
+
# Route a single sentence and inspect JSON telemetry
|
|
151
|
+
misar route "أعلنت منصة المطورين عن ترقية واجهة البرمجة API لدعم نماذج المحولات" --json
|
|
152
|
+
|
|
153
|
+
# Run the reproducible 15-sample benchmark suite
|
|
154
|
+
misar bench
|
|
155
|
+
|
|
156
|
+
# Launch the interactive RTL Web Workbench & hardened REST API on http://127.0.0.1:8050
|
|
157
|
+
misar serve
|
|
158
|
+
```
|
|
159
|
+
|
|
160
|
+
### Embedded REST API Endpoints (`misar serve`)
|
|
161
|
+
|
|
162
|
+
| Method | Endpoint | Description |
|
|
163
|
+
| :--- | :--- | :--- |
|
|
164
|
+
| `GET` | `/api/health` | Service status, version, and active tokenizer backend |
|
|
165
|
+
| `GET` | `/api/suite` | Evaluates the 6 standard cross-domain reference samples |
|
|
166
|
+
| `POST` | `/api/route` | Body `{"text": "..."}` → Returns full `RoutingResult` telemetry |
|
|
167
|
+
| `POST` | `/api/batch` | Body `{"texts": ["...", "..."]}` → Batch routes up to 500 strings (`1 MiB` payload limit) |
|
|
168
|
+
|
|
169
|
+
---
|
|
170
|
+
|
|
171
|
+
## Repository Structure
|
|
172
|
+
|
|
173
|
+
```text
|
|
174
|
+
misar/
|
|
175
|
+
├── .github/workflows/ci.yml # Cross-OS Python 3.9–3.13 CI pipeline
|
|
176
|
+
├── docs/misar-architecture.png # High-resolution system architecture diagram
|
|
177
|
+
├── benchmarks/
|
|
178
|
+
│ ├── run_benchmark.py # Reproducible benchmark runner
|
|
179
|
+
│ └── latest_benchmark_report.json # Verified telemetry snapshot
|
|
180
|
+
├── src/misar/
|
|
181
|
+
│ ├── __init__.py # Public API exports
|
|
182
|
+
│ ├── profiler.py # O(n) 5-signal LinguisticProfiler & Shannon entropy
|
|
183
|
+
│ ├── lexicons.py # MSA-safe multi-region dialect & negation lexicons
|
|
184
|
+
│ ├── pipelines.py # FAST_MSA, DIALECT_PRESERVE & NOISY_SOCIAL passes
|
|
185
|
+
│ ├── tokenizer.py # Pure-Python WordPieceTokenizer + HF & offline adapters
|
|
186
|
+
│ ├── router.py # MisarRouter orchestration & wrap_hf_pipeline
|
|
187
|
+
│ ├── benchmark.py # Shared benchmark suite module
|
|
188
|
+
│ ├── cli.py # `misar` CLI (`demo`, `route`, `bench`, `serve`)
|
|
189
|
+
│ └── web/
|
|
190
|
+
│ ├── server.py # Hardened zero-dependency ThreadingHTTPServer REST API
|
|
191
|
+
│ └── index.html # Interactive Arabic RTL Workbench & token visualizer
|
|
192
|
+
├── tests/test_misar.py # 23 unit, security, server & real-vocab tests
|
|
193
|
+
├── CITATION.cff # Native GitHub repository citation metadata
|
|
194
|
+
├── CONTRIBUTING.md # Developer setup & contribution standards
|
|
195
|
+
├── SECURITY.md # Built-in API & frontend hardening controls
|
|
196
|
+
├── CHANGELOG.md # Formal release history
|
|
197
|
+
├── LICENSE # MIT License
|
|
198
|
+
└── pyproject.toml # PEP 621 PyPI package specification
|
|
199
|
+
```
|
|
200
|
+
|
|
201
|
+
---
|
|
202
|
+
|
|
203
|
+
## Testing & Verification
|
|
204
|
+
|
|
205
|
+
```bash
|
|
206
|
+
python -m unittest discover -s tests -v
|
|
207
|
+
```
|
|
208
|
+
|
|
209
|
+
All **23 automated tests** verify profiler mathematics, MSA homograph false-positive resistance, Saudi/Egyptian/Levantine/Maghrebi negation preservation, gemination protection (`والللله → والله`), markdown stripping, tatweel/diacritic splitting, authentic `AraBERTv2` WordPiece reduction, and REST server HTTP security bounds.
|
|
210
|
+
|
|
211
|
+
---
|
|
212
|
+
|
|
213
|
+
## Citation
|
|
214
|
+
|
|
215
|
+
If you use **Misar** in academic research, benchmarks, or commercial NLP pipelines, please cite:
|
|
216
|
+
|
|
217
|
+
```bibtex
|
|
218
|
+
@software{altheeb2026misar,
|
|
219
|
+
author = {Muneer Jarab Matrak Al-Theeb},
|
|
220
|
+
title = {Misar (مِسار): Adaptive Context-Aware Arabic NLP Preprocessing Router for Transformer Language Models},
|
|
221
|
+
year = {2026},
|
|
222
|
+
version = {1.0.0},
|
|
223
|
+
url = {https://github.com/RealMtrx/misar},
|
|
224
|
+
license = {MIT}
|
|
225
|
+
}
|
|
226
|
+
```
|
|
227
|
+
|
|
228
|
+
---
|
|
229
|
+
|
|
230
|
+
## 📖 نظرة عامة بالعربية
|
|
231
|
+
|
|
232
|
+
**مِسار (`Misar`)** هي مكتبة وموجّه بايثون مفتوح المصدر وخفيف الوزن (بلا أي تبعيات خارجية إجبارية) صُمِّم ليعمل كطبقة بوابة ذكية في زمن خطي $O(n)$ (**~0.22 مللي ثانية** للنص) قبل نماذج المحولات اللغوية العربية مثل `AraBERTv2` و `CAMeLBERT` والنماذج التوليدية (`ALLaM` و `LLMs`) **دون الحاجة إلى إعادة تدريب النموذج (`Zero Parameter Update`)**.
|
|
233
|
+
|
|
234
|
+
* **توجيه ثلاثي المسارات:** يفحص النص في تمريرة واحدة ويختار بين مسار الفصحى الحافظ للإملاء (`FAST_MSA`)، أو مسار حفظ اللهجة وأدوات النفي (`DIALECT_PRESERVE`) لحماية كلمات مثل `مو، مب، مافيه، ماكو`، أو مسار تنقية ضجيج شبكات التواصل (`NOISY_SOCIAL`) الذي يضغط المدود والتطويل مع الحفاظ على الكلمات المشددة (`والللله ← والله`).
|
|
235
|
+
* **توفير التوكنات وتكاليف الـ APIs:** أظهرت التجارب المرجعية الموثقة على معجم `AraBERTv2` خفض إجمالي التوكنات بنسبة **46.57%** مقارنة بالنص الخام و**26.33%** مقارنة بمناهج التطبيع القديمة، مع استعادة **100%** من الكلمات المتأثرة بالتشكيل والكشيدة (`156 → 0 [UNK]`).
|
|
236
|
+
|
|
237
|
+
---
|
|
238
|
+
|
|
239
|
+
<div align="center">
|
|
240
|
+
|
|
241
|
+
Authored and maintained by **Muneer Jarab Matrak Al-Theeb (منير جراب مترك الذيب)**
|
|
242
|
+
Released under the [MIT License](https://github.com/RealMtrx/misar/blob/main/LICENSE) © 2026
|
|
243
|
+
|
|
244
|
+
</div>
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=68.0", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "misar"
|
|
7
|
+
version = "1.0.0"
|
|
8
|
+
description = "Misar (مِسار): Adaptive context-aware Arabic NLP preprocessing router for optimizing Transformer language models."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = {text = "MIT"}
|
|
11
|
+
requires-python = ">=3.9"
|
|
12
|
+
authors = [
|
|
13
|
+
{name = "Muneer Jarab Matrak Al-Theeb"}
|
|
14
|
+
]
|
|
15
|
+
keywords = [
|
|
16
|
+
"arabic",
|
|
17
|
+
"nlp",
|
|
18
|
+
"preprocessing",
|
|
19
|
+
"router",
|
|
20
|
+
"arabert",
|
|
21
|
+
"camelbert",
|
|
22
|
+
"llm",
|
|
23
|
+
"tokenization",
|
|
24
|
+
"dialect",
|
|
25
|
+
"saudi-arabia"
|
|
26
|
+
]
|
|
27
|
+
classifiers = [
|
|
28
|
+
"Development Status :: 5 - Production/Stable",
|
|
29
|
+
"Intended Audience :: Developers",
|
|
30
|
+
"Intended Audience :: Science/Research",
|
|
31
|
+
"License :: OSI Approved :: MIT License",
|
|
32
|
+
"Natural Language :: Arabic",
|
|
33
|
+
"Operating System :: OS Independent",
|
|
34
|
+
"Programming Language :: Python :: 3",
|
|
35
|
+
"Programming Language :: Python :: 3.9",
|
|
36
|
+
"Programming Language :: Python :: 3.10",
|
|
37
|
+
"Programming Language :: Python :: 3.11",
|
|
38
|
+
"Programming Language :: Python :: 3.12",
|
|
39
|
+
"Programming Language :: Python :: 3.13",
|
|
40
|
+
"Programming Language :: Python :: 3.14",
|
|
41
|
+
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
|
42
|
+
"Topic :: Text Processing :: Linguistic",
|
|
43
|
+
]
|
|
44
|
+
dependencies = []
|
|
45
|
+
|
|
46
|
+
[project.optional-dependencies]
|
|
47
|
+
huggingface = ["transformers>=4.35.0", "torch>=2.0.0"]
|
|
48
|
+
dev = ["pytest>=7.4.0", "build>=1.0.0", "twine>=5.0.0"]
|
|
49
|
+
|
|
50
|
+
[project.scripts]
|
|
51
|
+
misar = "misar.cli:main"
|
|
52
|
+
|
|
53
|
+
[project.urls]
|
|
54
|
+
Homepage = "https://github.com/RealMtrx/misar"
|
|
55
|
+
Repository = "https://github.com/RealMtrx/misar"
|
|
56
|
+
Issues = "https://github.com/RealMtrx/misar/issues"
|
|
57
|
+
|
|
58
|
+
[tool.setuptools.packages.find]
|
|
59
|
+
where = ["src"]
|
|
60
|
+
|
|
61
|
+
[tool.setuptools.package-data]
|
|
62
|
+
misar = ["web/*.html"]
|
misar-1.0.0/setup.cfg
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
"""
|
|
3
|
+
Misar (مِسار): Adaptive Context-Aware Arabic NLP Preprocessing Router.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from .pipelines import DialectPreservePipeline, MSALightPipeline, NoisySocialPipeline
|
|
7
|
+
from .profiler import LinguisticProfile, LinguisticProfiler, compute_shannon_entropy
|
|
8
|
+
from .router import MisarRouter, RoutingResult, preprocess, route
|
|
9
|
+
from .tokenizer import (
|
|
10
|
+
HeuristicSubwordTokenizer,
|
|
11
|
+
HuggingFaceTokenizerAdapter,
|
|
12
|
+
WordPieceTokenizer,
|
|
13
|
+
load_default_tokenizer,
|
|
14
|
+
)
|
|
15
|
+
|
|
16
|
+
__version__ = "1.0.0"
|
|
17
|
+
__author__ = "Muneer Jarab Matrak Al-Theeb"
|
|
18
|
+
|
|
19
|
+
__all__ = [
|
|
20
|
+
"__version__",
|
|
21
|
+
"__author__",
|
|
22
|
+
"MisarRouter",
|
|
23
|
+
"RoutingResult",
|
|
24
|
+
"LinguisticProfiler",
|
|
25
|
+
"LinguisticProfile",
|
|
26
|
+
"compute_shannon_entropy",
|
|
27
|
+
"MSALightPipeline",
|
|
28
|
+
"DialectPreservePipeline",
|
|
29
|
+
"NoisySocialPipeline",
|
|
30
|
+
"WordPieceTokenizer",
|
|
31
|
+
"HuggingFaceTokenizerAdapter",
|
|
32
|
+
"HeuristicSubwordTokenizer",
|
|
33
|
+
"load_default_tokenizer",
|
|
34
|
+
"route",
|
|
35
|
+
"preprocess",
|
|
36
|
+
]
|