misar 1.0.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
misar-1.0.0/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Muneer Jarab Matrak Al-Theeb — Misar NLP
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
misar-1.0.0/PKG-INFO ADDED
@@ -0,0 +1,281 @@
1
+ Metadata-Version: 2.4
2
+ Name: misar
3
+ Version: 1.0.0
4
+ Summary: Misar (مِسار): Adaptive context-aware Arabic NLP preprocessing router for optimizing Transformer language models.
5
+ Author: Muneer Jarab Matrak Al-Theeb
6
+ License: MIT
7
+ Project-URL: Homepage, https://github.com/RealMtrx/misar
8
+ Project-URL: Repository, https://github.com/RealMtrx/misar
9
+ Project-URL: Issues, https://github.com/RealMtrx/misar/issues
10
+ Keywords: arabic,nlp,preprocessing,router,arabert,camelbert,llm,tokenization,dialect,saudi-arabia
11
+ Classifier: Development Status :: 5 - Production/Stable
12
+ Classifier: Intended Audience :: Developers
13
+ Classifier: Intended Audience :: Science/Research
14
+ Classifier: License :: OSI Approved :: MIT License
15
+ Classifier: Natural Language :: Arabic
16
+ Classifier: Operating System :: OS Independent
17
+ Classifier: Programming Language :: Python :: 3
18
+ Classifier: Programming Language :: Python :: 3.9
19
+ Classifier: Programming Language :: Python :: 3.10
20
+ Classifier: Programming Language :: Python :: 3.11
21
+ Classifier: Programming Language :: Python :: 3.12
22
+ Classifier: Programming Language :: Python :: 3.13
23
+ Classifier: Programming Language :: Python :: 3.14
24
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
25
+ Classifier: Topic :: Text Processing :: Linguistic
26
+ Requires-Python: >=3.9
27
+ Description-Content-Type: text/markdown
28
+ License-File: LICENSE
29
+ Provides-Extra: huggingface
30
+ Requires-Dist: transformers>=4.35.0; extra == "huggingface"
31
+ Requires-Dist: torch>=2.0.0; extra == "huggingface"
32
+ Provides-Extra: dev
33
+ Requires-Dist: pytest>=7.4.0; extra == "dev"
34
+ Requires-Dist: build>=1.0.0; extra == "dev"
35
+ Requires-Dist: twine>=5.0.0; extra == "dev"
36
+ Dynamic: license-file
37
+
38
+ <div align="center">
39
+
40
+ # مِسار — Misar
41
+
42
+ **Adaptive $O(n)$ Context-Aware Arabic NLP Preprocessing Router for Transformer Language Models**
43
+ **الموجّه اللغوي التكيفي لتعزيز كفاءة ودقة معالجة اللغة العربية في نماذج الذكاء الاصطناعي**
44
+
45
+ [![CI](https://github.com/RealMtrx/misar/actions/workflows/ci.yml/badge.svg)](https://github.com/RealMtrx/misar/actions/workflows/ci.yml)
46
+ [![Python 3.9+](https://img.shields.io/badge/python-3.9%2B-1e40af.svg)](https://www.python.org/downloads/)
47
+ [![Dependencies](https://img.shields.io/badge/dependencies-zero%20mandatory-15803d.svg)](#installation)
48
+ [![License: MIT](https://img.shields.io/badge/License-MIT-0f2942.svg)](https://github.com/RealMtrx/misar/blob/main/LICENSE)
49
+ [![Latency](https://img.shields.io/badge/CPU%20latency-0.22%20ms%2Ftext-0369a1.svg)](#empirical-benchmarks)
50
+ [![Token Reduction](https://img.shields.io/badge/tokens%20saved--46.57%25-16a34a.svg)](#empirical-benchmarks)
51
+
52
+ [Architecture](#architecture) • [Benchmarks](#empirical-benchmarks) • [Installation](#installation) • [Quick Start](#quick-start) • [CLI & Workbench](#cli--web-workbench) • [Citation](#citation) • [النسخة العربية](#-نظرة-عامة-بالعربية)
53
+
54
+ <br/>
55
+
56
+ <img src="https://raw.githubusercontent.com/RealMtrx/misar/main/docs/misar-architecture.png" alt="Misar Adaptive Arabic NLP Preprocessing Router Architecture" width="100%" />
57
+
58
+ </div>
59
+
60
+ ---
61
+
62
+ ## Why Misar?
63
+
64
+ Modern Arabic NLP deployments face a structural bottleneck: forcing heterogeneous Arabic text (**Modern Standard Arabic**, **classical diacritized prose**, **Gulf/Egyptian/Levantine/Maghrebi dialects**, and **noisy social media**) through a single static preprocessing pipeline actively degrades both accuracy and inference economics:
65
+
66
+ 1. **Legacy Normalization Damages Modern Transformers:** Classical regex pipelines blindly collapse `ة → ه` and `أ/إ/آ → ا`. However, modern Transformer vocabularies such as **AraBERTv2** (`64k` WordPiece) and **CAMeLBERT** explicitly preserve Hamzas and Taa Marbuta—meaning legacy normalization shatters known whole words (e.g., `الحوسبة` as `1` token splits into `الحو` + `##سبه` as `2` tokens).
67
+ 2. **Dialectal Sentiment & Negation Erasure:** Over-aggressive cleaning distorts critical colloquial negation and sentiment markers (`مو`, `مب`, `مافيه`, `ماكو`), reversing downstream classification polarity.
68
+ 3. **Diacritic, Tatweel & Elongation Token Explosion:** Unstripped harakat (`تشكيل`), kashida (`ـ`), and character stretches (`بطييييء`, `السيرفررررر`) trigger OOV `[UNK]` blindness or inflate subword token counts—multiplying Transformer $O(L^2)$ self-attention cost and API billing.
69
+
70
+ **Misar (`مِسار`)** solves all three challenges **without retraining any model (`Zero Parameter Update`)** by acting as an ultra-lightweight, zero-dependency, linear-time $O(n)$ routing gateway directly in front of your Transformer or LLM pipeline.
71
+
72
+ ---
73
+
74
+ ## Architecture
75
+
76
+ Every incoming text passes once through `LinguisticProfiler` (`< 0.25 ms` on CPU) to compute five quantitative signals (`dialect_ratio` $P_{\text{dial}}$, `diacritic_density` $D_{\text{diac}}$, `elongation_noise_score` $E_{\text{noise}}$, character Shannon Entropy $H_{\text{char}}$, and `code_switching_ratio` $R_{\text{latin}}$), then dispatches dynamically to the optimal micro-pipeline:
77
+
78
+ ```mermaid
79
+ flowchart LR
80
+ A["Raw Arabic Input<br/>(MSA / Diacritized / Dialect / Social)"] --> B["LinguisticProfiler O(n)<br/>P_dial · D_diac · E_noise · H_char · R_latin"]
81
+ B -->|"High Noise / Tatweel / Stretches ≥ 3"| C["NOISY_SOCIAL (+ Hybrid Dialect)<br/>Gemination-Safe Collapse + Tag Unpacking"]
82
+ B -->|"Dialect Markers / Protected Negations"| D["DIALECT_PRESERVE<br/>Negation Protection + Vocab-Aligned Cleanup"]
83
+ B -->|"Clean MSA / Classical Diacritized"| E["FAST_MSA<br/>Strip Harakat · Preserve أ/إ/آ, ة, ى"]
84
+ C --> F["AraBERTv2 / CAMeLBERT / ALLaM / LLMs<br/>(-46.57% Tokens · 0 [UNK])"]
85
+ D --> F
86
+ E --> F
87
+ ```
88
+
89
+ ---
90
+
91
+ ## Empirical Benchmarks
92
+
93
+ Measured reproducibly via `python benchmarks/run_benchmark.py` against the authentic **64,000-token `aubmindlab/bert-base-arabertv02` WordPiece vocabulary** across 15 curated multi-domain samples ([`latest_benchmark_report.json`](https://github.com/RealMtrx/misar/blob/main/benchmarks/latest_benchmark_report.json)):
94
+
95
+ | Domain Category | Samples | Raw Tokens | Legacy Static (`ة→ه`, `أ→ا`) | **Misar Adaptive** | Misar Gain vs. Raw |
96
+ | :--- | :---: | :---: | :---: | :---: | :---: |
97
+ | **MSA Engineering & Code-Switch** | 3 | 57 | 79 *(+38.6% fragmentation)* | **57** | Preserves `100%` orthography (`-27.85%` vs Legacy) |
98
+ | **Classical Diacritized (Heritage)** | 3 | 186 *(147 OOV `[UNK]`)* | 48 | **39** | **`-79.03%`** tokens & **`100%`** word recovery (`0 [UNK]`) |
99
+ | **Saudi / Gulf / Levantine / Maghrebi Dev** | 6 | 121 | 131 | **117** | **`-3.31%`** tokens (`-10.69%` vs Legacy) + negations intact |
100
+ | **Noisy Tech Social & Code-Switching** | 3 | 102 | 80 | **36** | **`-64.71%`** tokens + gemination preserved |
101
+ | **Full Corpus Summary** | **15** | **466** | **338** | **249** | **`-46.57%` vs Raw · `-26.33%` vs Legacy · `0.223 ms/text`** |
102
+
103
+ > **Key Scientific Proof:** On Standard Arabic technical prose (`MSA Tech / Engineering`), blind legacy normalization inflated token count from **57 → 79** due to `ة → ه` subword splitting, while Misar maintained **57** intact vocabulary tokens (`TFR ≈ 1.0`). Across all 15 multi-domain inputs, Misar saved **46.57%** of total tokens vs. raw text and **26.33%** vs. legacy normalization (recovering **156 → 0 `[UNK]`** tokens) with **0.223 ms** mean latency on pure CPU.
104
+
105
+ ---
106
+
107
+ ## Installation
108
+
109
+ Core `misar` requires **zero mandatory external packages** (Python `3.9+` Standard Library only):
110
+
111
+ ```bash
112
+ # Install directly from GitHub
113
+ pip install git+https://github.com/RealMtrx/misar.git
114
+
115
+ # Or from PyPI once published
116
+ pip install misar
117
+ ```
118
+
119
+ Optional extras for live Hugging Face Transformers wrapping and package development:
120
+
121
+ ```bash
122
+ pip install "misar[huggingface]" # transformers + torch
123
+ pip install "misar[dev]" # pytest + build + twine
124
+ ```
125
+
126
+ *(Note: On first use of the built-in `WordPieceTokenizer`, Misar caches the official 64k vocabulary file locally under `~/.cache/misar/`. Pass `allow_vocab_download=False` or `--offline` to run completely air-gapped using `HeuristicSubwordTokenizer`.)*
127
+
128
+ ---
129
+
130
+ ## Quick Start
131
+
132
+ ### 1. Dynamic Text Routing & Token Telemetry
133
+
134
+ ```python
135
+ from misar import MisarRouter
136
+
137
+ router = MisarRouter() # Uses real AraBERTv2 64k WordPiece vocabulary
138
+
139
+ result = router.route("السيرفررررر بطييييء مررررة ومافيه استجابة في API أبدددداً 😡😡👎 #هندسة_البرمجيات")
140
+
141
+ print("Route: ", result.route_name) # NOISY_SOCIAL
142
+ print("Processed: ", result.processed_text) # السيرفر بطيء مرة ومافيه استجابة في API أبدا هندسة البرمجيات
143
+ print("Tokens: ", result.raw_token_count, "->", result.processed_token_count) # 28 -> 14
144
+ print("Reduction: ", f"-{result.token_reduction_pct:.1f}%") # -50.0%
145
+ print("Latency: ", f"{result.latency_ms:.3f} ms")
146
+ ```
147
+
148
+ ### 2. Zero-Latency Drop-In String Preprocessing & Batching
149
+
150
+ ```python
151
+ from misar import MisarRouter, preprocess
152
+
153
+ # One-line module helper
154
+ clean = preprocess("الســــلام علــــيكم ومافيه مشكلة #تطوير_البرمجيات")
155
+ # -> "السلام عليكم ومافيه مشكلة تطوير البرمجيات"
156
+
157
+ # Fast batch routing for production ETL pipelines
158
+ router = MisarRouter()
159
+ results = router.route_batch([
160
+ "أصدرت عدة دول عربية بياناً مشتركاً لأول مرة عن صافي الأرباح", # -> FAST_MSA (zero dialect false positives)
161
+ "والللله الكود مو معقد أبداً والأخطاء متكررررة", # -> NOISY_SOCIAL (preserves والله and مو)
162
+ ])
163
+ ```
164
+
165
+ ### 3. One-Line Hugging Face Transformers Integration
166
+
167
+ ```python
168
+ from transformers import pipeline
169
+ from misar import MisarRouter
170
+
171
+ sentiment = pipeline("sentiment-analysis", model="CAMeL-Lab/bert-base-arabic-camelbert-mix-sentiment")
172
+ smart_sentiment = MisarRouter(tokenizer=sentiment.tokenizer).wrap_hf_pipeline(sentiment)
173
+
174
+ print(smart_sentiment("الكود مو معقد أبداً بس السيرفررررر بطييييء #مشكلة"))
175
+ ```
176
+
177
+ ---
178
+
179
+ ## CLI & Web Workbench
180
+
181
+ Misar includes a clean command-line interface and a self-hosted HTTP server + Arabic RTL Interactive Workbench:
182
+
183
+ ```bash
184
+ # Run 6 built-in cross-dialect samples with live AraBERTv2 tokenization
185
+ misar demo
186
+
187
+ # Route a single sentence and inspect JSON telemetry
188
+ misar route "أعلنت منصة المطورين عن ترقية واجهة البرمجة API لدعم نماذج المحولات" --json
189
+
190
+ # Run the reproducible 15-sample benchmark suite
191
+ misar bench
192
+
193
+ # Launch the interactive RTL Web Workbench & hardened REST API on http://127.0.0.1:8050
194
+ misar serve
195
+ ```
196
+
197
+ ### Embedded REST API Endpoints (`misar serve`)
198
+
199
+ | Method | Endpoint | Description |
200
+ | :--- | :--- | :--- |
201
+ | `GET` | `/api/health` | Service status, version, and active tokenizer backend |
202
+ | `GET` | `/api/suite` | Evaluates the 6 standard cross-domain reference samples |
203
+ | `POST` | `/api/route` | Body `{"text": "..."}` → Returns full `RoutingResult` telemetry |
204
+ | `POST` | `/api/batch` | Body `{"texts": ["...", "..."]}` → Batch routes up to 500 strings (`1 MiB` payload limit) |
205
+
206
+ ---
207
+
208
+ ## Repository Structure
209
+
210
+ ```text
211
+ misar/
212
+ ├── .github/workflows/ci.yml # Cross-OS Python 3.9–3.13 CI pipeline
213
+ ├── docs/misar-architecture.png # High-resolution system architecture diagram
214
+ ├── benchmarks/
215
+ │ ├── run_benchmark.py # Reproducible benchmark runner
216
+ │ └── latest_benchmark_report.json # Verified telemetry snapshot
217
+ ├── src/misar/
218
+ │ ├── __init__.py # Public API exports
219
+ │ ├── profiler.py # O(n) 5-signal LinguisticProfiler & Shannon entropy
220
+ │ ├── lexicons.py # MSA-safe multi-region dialect & negation lexicons
221
+ │ ├── pipelines.py # FAST_MSA, DIALECT_PRESERVE & NOISY_SOCIAL passes
222
+ │ ├── tokenizer.py # Pure-Python WordPieceTokenizer + HF & offline adapters
223
+ │ ├── router.py # MisarRouter orchestration & wrap_hf_pipeline
224
+ │ ├── benchmark.py # Shared benchmark suite module
225
+ │ ├── cli.py # `misar` CLI (`demo`, `route`, `bench`, `serve`)
226
+ │ └── web/
227
+ │ ├── server.py # Hardened zero-dependency ThreadingHTTPServer REST API
228
+ │ └── index.html # Interactive Arabic RTL Workbench & token visualizer
229
+ ├── tests/test_misar.py # 23 unit, security, server & real-vocab tests
230
+ ├── CITATION.cff # Native GitHub repository citation metadata
231
+ ├── CONTRIBUTING.md # Developer setup & contribution standards
232
+ ├── SECURITY.md # Built-in API & frontend hardening controls
233
+ ├── CHANGELOG.md # Formal release history
234
+ ├── LICENSE # MIT License
235
+ └── pyproject.toml # PEP 621 PyPI package specification
236
+ ```
237
+
238
+ ---
239
+
240
+ ## Testing & Verification
241
+
242
+ ```bash
243
+ python -m unittest discover -s tests -v
244
+ ```
245
+
246
+ All **23 automated tests** verify profiler mathematics, MSA homograph false-positive resistance, Saudi/Egyptian/Levantine/Maghrebi negation preservation, gemination protection (`والللله → والله`), markdown stripping, tatweel/diacritic splitting, authentic `AraBERTv2` WordPiece reduction, and REST server HTTP security bounds.
247
+
248
+ ---
249
+
250
+ ## Citation
251
+
252
+ If you use **Misar** in academic research, benchmarks, or commercial NLP pipelines, please cite:
253
+
254
+ ```bibtex
255
+ @software{altheeb2026misar,
256
+ author = {Muneer Jarab Matrak Al-Theeb},
257
+ title = {Misar (مِسار): Adaptive Context-Aware Arabic NLP Preprocessing Router for Transformer Language Models},
258
+ year = {2026},
259
+ version = {1.0.0},
260
+ url = {https://github.com/RealMtrx/misar},
261
+ license = {MIT}
262
+ }
263
+ ```
264
+
265
+ ---
266
+
267
+ ## 📖 نظرة عامة بالعربية
268
+
269
+ **مِسار (`Misar`)** هي مكتبة وموجّه بايثون مفتوح المصدر وخفيف الوزن (بلا أي تبعيات خارجية إجبارية) صُمِّم ليعمل كطبقة بوابة ذكية في زمن خطي $O(n)$ (**~0.22 مللي ثانية** للنص) قبل نماذج المحولات اللغوية العربية مثل `AraBERTv2` و `CAMeLBERT` والنماذج التوليدية (`ALLaM` و `LLMs`) **دون الحاجة إلى إعادة تدريب النموذج (`Zero Parameter Update`)**.
270
+
271
+ * **توجيه ثلاثي المسارات:** يفحص النص في تمريرة واحدة ويختار بين مسار الفصحى الحافظ للإملاء (`FAST_MSA`)، أو مسار حفظ اللهجة وأدوات النفي (`DIALECT_PRESERVE`) لحماية كلمات مثل `مو، مب، مافيه، ماكو`، أو مسار تنقية ضجيج شبكات التواصل (`NOISY_SOCIAL`) الذي يضغط المدود والتطويل مع الحفاظ على الكلمات المشددة (`والللله ← والله`).
272
+ * **توفير التوكنات وتكاليف الـ APIs:** أظهرت التجارب المرجعية الموثقة على معجم `AraBERTv2` خفض إجمالي التوكنات بنسبة **46.57%** مقارنة بالنص الخام و**26.33%** مقارنة بمناهج التطبيع القديمة، مع استعادة **100%** من الكلمات المتأثرة بالتشكيل والكشيدة (`156 → 0 [UNK]`).
273
+
274
+ ---
275
+
276
+ <div align="center">
277
+
278
+ Authored and maintained by **Muneer Jarab Matrak Al-Theeb (منير جراب مترك الذيب)**
279
+ Released under the [MIT License](https://github.com/RealMtrx/misar/blob/main/LICENSE) © 2026
280
+
281
+ </div>
misar-1.0.0/README.md ADDED
@@ -0,0 +1,244 @@
1
+ <div align="center">
2
+
3
+ # مِسار — Misar
4
+
5
+ **Adaptive $O(n)$ Context-Aware Arabic NLP Preprocessing Router for Transformer Language Models**
6
+ **الموجّه اللغوي التكيفي لتعزيز كفاءة ودقة معالجة اللغة العربية في نماذج الذكاء الاصطناعي**
7
+
8
+ [![CI](https://github.com/RealMtrx/misar/actions/workflows/ci.yml/badge.svg)](https://github.com/RealMtrx/misar/actions/workflows/ci.yml)
9
+ [![Python 3.9+](https://img.shields.io/badge/python-3.9%2B-1e40af.svg)](https://www.python.org/downloads/)
10
+ [![Dependencies](https://img.shields.io/badge/dependencies-zero%20mandatory-15803d.svg)](#installation)
11
+ [![License: MIT](https://img.shields.io/badge/License-MIT-0f2942.svg)](https://github.com/RealMtrx/misar/blob/main/LICENSE)
12
+ [![Latency](https://img.shields.io/badge/CPU%20latency-0.22%20ms%2Ftext-0369a1.svg)](#empirical-benchmarks)
13
+ [![Token Reduction](https://img.shields.io/badge/tokens%20saved--46.57%25-16a34a.svg)](#empirical-benchmarks)
14
+
15
+ [Architecture](#architecture) • [Benchmarks](#empirical-benchmarks) • [Installation](#installation) • [Quick Start](#quick-start) • [CLI & Workbench](#cli--web-workbench) • [Citation](#citation) • [النسخة العربية](#-نظرة-عامة-بالعربية)
16
+
17
+ <br/>
18
+
19
+ <img src="https://raw.githubusercontent.com/RealMtrx/misar/main/docs/misar-architecture.png" alt="Misar Adaptive Arabic NLP Preprocessing Router Architecture" width="100%" />
20
+
21
+ </div>
22
+
23
+ ---
24
+
25
+ ## Why Misar?
26
+
27
+ Modern Arabic NLP deployments face a structural bottleneck: forcing heterogeneous Arabic text (**Modern Standard Arabic**, **classical diacritized prose**, **Gulf/Egyptian/Levantine/Maghrebi dialects**, and **noisy social media**) through a single static preprocessing pipeline actively degrades both accuracy and inference economics:
28
+
29
+ 1. **Legacy Normalization Damages Modern Transformers:** Classical regex pipelines blindly collapse `ة → ه` and `أ/إ/آ → ا`. However, modern Transformer vocabularies such as **AraBERTv2** (`64k` WordPiece) and **CAMeLBERT** explicitly preserve Hamzas and Taa Marbuta—meaning legacy normalization shatters known whole words (e.g., `الحوسبة` as `1` token splits into `الحو` + `##سبه` as `2` tokens).
30
+ 2. **Dialectal Sentiment & Negation Erasure:** Over-aggressive cleaning distorts critical colloquial negation and sentiment markers (`مو`, `مب`, `مافيه`, `ماكو`), reversing downstream classification polarity.
31
+ 3. **Diacritic, Tatweel & Elongation Token Explosion:** Unstripped harakat (`تشكيل`), kashida (`ـ`), and character stretches (`بطييييء`, `السيرفررررر`) trigger OOV `[UNK]` blindness or inflate subword token counts—multiplying Transformer $O(L^2)$ self-attention cost and API billing.
32
+
33
+ **Misar (`مِسار`)** solves all three challenges **without retraining any model (`Zero Parameter Update`)** by acting as an ultra-lightweight, zero-dependency, linear-time $O(n)$ routing gateway directly in front of your Transformer or LLM pipeline.
34
+
35
+ ---
36
+
37
+ ## Architecture
38
+
39
+ Every incoming text passes once through `LinguisticProfiler` (`< 0.25 ms` on CPU) to compute five quantitative signals (`dialect_ratio` $P_{\text{dial}}$, `diacritic_density` $D_{\text{diac}}$, `elongation_noise_score` $E_{\text{noise}}$, character Shannon Entropy $H_{\text{char}}$, and `code_switching_ratio` $R_{\text{latin}}$), then dispatches dynamically to the optimal micro-pipeline:
40
+
41
+ ```mermaid
42
+ flowchart LR
43
+ A["Raw Arabic Input<br/>(MSA / Diacritized / Dialect / Social)"] --> B["LinguisticProfiler O(n)<br/>P_dial · D_diac · E_noise · H_char · R_latin"]
44
+ B -->|"High Noise / Tatweel / Stretches ≥ 3"| C["NOISY_SOCIAL (+ Hybrid Dialect)<br/>Gemination-Safe Collapse + Tag Unpacking"]
45
+ B -->|"Dialect Markers / Protected Negations"| D["DIALECT_PRESERVE<br/>Negation Protection + Vocab-Aligned Cleanup"]
46
+ B -->|"Clean MSA / Classical Diacritized"| E["FAST_MSA<br/>Strip Harakat · Preserve أ/إ/آ, ة, ى"]
47
+ C --> F["AraBERTv2 / CAMeLBERT / ALLaM / LLMs<br/>(-46.57% Tokens · 0 [UNK])"]
48
+ D --> F
49
+ E --> F
50
+ ```
51
+
52
+ ---
53
+
54
+ ## Empirical Benchmarks
55
+
56
+ Measured reproducibly via `python benchmarks/run_benchmark.py` against the authentic **64,000-token `aubmindlab/bert-base-arabertv02` WordPiece vocabulary** across 15 curated multi-domain samples ([`latest_benchmark_report.json`](https://github.com/RealMtrx/misar/blob/main/benchmarks/latest_benchmark_report.json)):
57
+
58
+ | Domain Category | Samples | Raw Tokens | Legacy Static (`ة→ه`, `أ→ا`) | **Misar Adaptive** | Misar Gain vs. Raw |
59
+ | :--- | :---: | :---: | :---: | :---: | :---: |
60
+ | **MSA Engineering & Code-Switch** | 3 | 57 | 79 *(+38.6% fragmentation)* | **57** | Preserves `100%` orthography (`-27.85%` vs Legacy) |
61
+ | **Classical Diacritized (Heritage)** | 3 | 186 *(147 OOV `[UNK]`)* | 48 | **39** | **`-79.03%`** tokens & **`100%`** word recovery (`0 [UNK]`) |
62
+ | **Saudi / Gulf / Levantine / Maghrebi Dev** | 6 | 121 | 131 | **117** | **`-3.31%`** tokens (`-10.69%` vs Legacy) + negations intact |
63
+ | **Noisy Tech Social & Code-Switching** | 3 | 102 | 80 | **36** | **`-64.71%`** tokens + gemination preserved |
64
+ | **Full Corpus Summary** | **15** | **466** | **338** | **249** | **`-46.57%` vs Raw · `-26.33%` vs Legacy · `0.223 ms/text`** |
65
+
66
+ > **Key Scientific Proof:** On Standard Arabic technical prose (`MSA Tech / Engineering`), blind legacy normalization inflated token count from **57 → 79** due to `ة → ه` subword splitting, while Misar maintained **57** intact vocabulary tokens (`TFR ≈ 1.0`). Across all 15 multi-domain inputs, Misar saved **46.57%** of total tokens vs. raw text and **26.33%** vs. legacy normalization (recovering **156 → 0 `[UNK]`** tokens) with **0.223 ms** mean latency on pure CPU.
67
+
68
+ ---
69
+
70
+ ## Installation
71
+
72
+ Core `misar` requires **zero mandatory external packages** (Python `3.9+` Standard Library only):
73
+
74
+ ```bash
75
+ # Install directly from GitHub
76
+ pip install git+https://github.com/RealMtrx/misar.git
77
+
78
+ # Or from PyPI once published
79
+ pip install misar
80
+ ```
81
+
82
+ Optional extras for live Hugging Face Transformers wrapping and package development:
83
+
84
+ ```bash
85
+ pip install "misar[huggingface]" # transformers + torch
86
+ pip install "misar[dev]" # pytest + build + twine
87
+ ```
88
+
89
+ *(Note: On first use of the built-in `WordPieceTokenizer`, Misar caches the official 64k vocabulary file locally under `~/.cache/misar/`. Pass `allow_vocab_download=False` or `--offline` to run completely air-gapped using `HeuristicSubwordTokenizer`.)*
90
+
91
+ ---
92
+
93
+ ## Quick Start
94
+
95
+ ### 1. Dynamic Text Routing & Token Telemetry
96
+
97
+ ```python
98
+ from misar import MisarRouter
99
+
100
+ router = MisarRouter() # Uses real AraBERTv2 64k WordPiece vocabulary
101
+
102
+ result = router.route("السيرفررررر بطييييء مررررة ومافيه استجابة في API أبدددداً 😡😡👎 #هندسة_البرمجيات")
103
+
104
+ print("Route: ", result.route_name) # NOISY_SOCIAL
105
+ print("Processed: ", result.processed_text) # السيرفر بطيء مرة ومافيه استجابة في API أبدا هندسة البرمجيات
106
+ print("Tokens: ", result.raw_token_count, "->", result.processed_token_count) # 28 -> 14
107
+ print("Reduction: ", f"-{result.token_reduction_pct:.1f}%") # -50.0%
108
+ print("Latency: ", f"{result.latency_ms:.3f} ms")
109
+ ```
110
+
111
+ ### 2. Zero-Latency Drop-In String Preprocessing & Batching
112
+
113
+ ```python
114
+ from misar import MisarRouter, preprocess
115
+
116
+ # One-line module helper
117
+ clean = preprocess("الســــلام علــــيكم ومافيه مشكلة #تطوير_البرمجيات")
118
+ # -> "السلام عليكم ومافيه مشكلة تطوير البرمجيات"
119
+
120
+ # Fast batch routing for production ETL pipelines
121
+ router = MisarRouter()
122
+ results = router.route_batch([
123
+ "أصدرت عدة دول عربية بياناً مشتركاً لأول مرة عن صافي الأرباح", # -> FAST_MSA (zero dialect false positives)
124
+ "والللله الكود مو معقد أبداً والأخطاء متكررررة", # -> NOISY_SOCIAL (preserves والله and مو)
125
+ ])
126
+ ```
127
+
128
+ ### 3. One-Line Hugging Face Transformers Integration
129
+
130
+ ```python
131
+ from transformers import pipeline
132
+ from misar import MisarRouter
133
+
134
+ sentiment = pipeline("sentiment-analysis", model="CAMeL-Lab/bert-base-arabic-camelbert-mix-sentiment")
135
+ smart_sentiment = MisarRouter(tokenizer=sentiment.tokenizer).wrap_hf_pipeline(sentiment)
136
+
137
+ print(smart_sentiment("الكود مو معقد أبداً بس السيرفررررر بطييييء #مشكلة"))
138
+ ```
139
+
140
+ ---
141
+
142
+ ## CLI & Web Workbench
143
+
144
+ Misar includes a clean command-line interface and a self-hosted HTTP server + Arabic RTL Interactive Workbench:
145
+
146
+ ```bash
147
+ # Run 6 built-in cross-dialect samples with live AraBERTv2 tokenization
148
+ misar demo
149
+
150
+ # Route a single sentence and inspect JSON telemetry
151
+ misar route "أعلنت منصة المطورين عن ترقية واجهة البرمجة API لدعم نماذج المحولات" --json
152
+
153
+ # Run the reproducible 15-sample benchmark suite
154
+ misar bench
155
+
156
+ # Launch the interactive RTL Web Workbench & hardened REST API on http://127.0.0.1:8050
157
+ misar serve
158
+ ```
159
+
160
+ ### Embedded REST API Endpoints (`misar serve`)
161
+
162
+ | Method | Endpoint | Description |
163
+ | :--- | :--- | :--- |
164
+ | `GET` | `/api/health` | Service status, version, and active tokenizer backend |
165
+ | `GET` | `/api/suite` | Evaluates the 6 standard cross-domain reference samples |
166
+ | `POST` | `/api/route` | Body `{"text": "..."}` → Returns full `RoutingResult` telemetry |
167
+ | `POST` | `/api/batch` | Body `{"texts": ["...", "..."]}` → Batch routes up to 500 strings (`1 MiB` payload limit) |
168
+
169
+ ---
170
+
171
+ ## Repository Structure
172
+
173
+ ```text
174
+ misar/
175
+ ├── .github/workflows/ci.yml # Cross-OS Python 3.9–3.13 CI pipeline
176
+ ├── docs/misar-architecture.png # High-resolution system architecture diagram
177
+ ├── benchmarks/
178
+ │ ├── run_benchmark.py # Reproducible benchmark runner
179
+ │ └── latest_benchmark_report.json # Verified telemetry snapshot
180
+ ├── src/misar/
181
+ │ ├── __init__.py # Public API exports
182
+ │ ├── profiler.py # O(n) 5-signal LinguisticProfiler & Shannon entropy
183
+ │ ├── lexicons.py # MSA-safe multi-region dialect & negation lexicons
184
+ │ ├── pipelines.py # FAST_MSA, DIALECT_PRESERVE & NOISY_SOCIAL passes
185
+ │ ├── tokenizer.py # Pure-Python WordPieceTokenizer + HF & offline adapters
186
+ │ ├── router.py # MisarRouter orchestration & wrap_hf_pipeline
187
+ │ ├── benchmark.py # Shared benchmark suite module
188
+ │ ├── cli.py # `misar` CLI (`demo`, `route`, `bench`, `serve`)
189
+ │ └── web/
190
+ │ ├── server.py # Hardened zero-dependency ThreadingHTTPServer REST API
191
+ │ └── index.html # Interactive Arabic RTL Workbench & token visualizer
192
+ ├── tests/test_misar.py # 23 unit, security, server & real-vocab tests
193
+ ├── CITATION.cff # Native GitHub repository citation metadata
194
+ ├── CONTRIBUTING.md # Developer setup & contribution standards
195
+ ├── SECURITY.md # Built-in API & frontend hardening controls
196
+ ├── CHANGELOG.md # Formal release history
197
+ ├── LICENSE # MIT License
198
+ └── pyproject.toml # PEP 621 PyPI package specification
199
+ ```
200
+
201
+ ---
202
+
203
+ ## Testing & Verification
204
+
205
+ ```bash
206
+ python -m unittest discover -s tests -v
207
+ ```
208
+
209
+ All **23 automated tests** verify profiler mathematics, MSA homograph false-positive resistance, Saudi/Egyptian/Levantine/Maghrebi negation preservation, gemination protection (`والللله → والله`), markdown stripping, tatweel/diacritic splitting, authentic `AraBERTv2` WordPiece reduction, and REST server HTTP security bounds.
210
+
211
+ ---
212
+
213
+ ## Citation
214
+
215
+ If you use **Misar** in academic research, benchmarks, or commercial NLP pipelines, please cite:
216
+
217
+ ```bibtex
218
+ @software{altheeb2026misar,
219
+ author = {Muneer Jarab Matrak Al-Theeb},
220
+ title = {Misar (مِسار): Adaptive Context-Aware Arabic NLP Preprocessing Router for Transformer Language Models},
221
+ year = {2026},
222
+ version = {1.0.0},
223
+ url = {https://github.com/RealMtrx/misar},
224
+ license = {MIT}
225
+ }
226
+ ```
227
+
228
+ ---
229
+
230
+ ## 📖 نظرة عامة بالعربية
231
+
232
+ **مِسار (`Misar`)** هي مكتبة وموجّه بايثون مفتوح المصدر وخفيف الوزن (بلا أي تبعيات خارجية إجبارية) صُمِّم ليعمل كطبقة بوابة ذكية في زمن خطي $O(n)$ (**~0.22 مللي ثانية** للنص) قبل نماذج المحولات اللغوية العربية مثل `AraBERTv2` و `CAMeLBERT` والنماذج التوليدية (`ALLaM` و `LLMs`) **دون الحاجة إلى إعادة تدريب النموذج (`Zero Parameter Update`)**.
233
+
234
+ * **توجيه ثلاثي المسارات:** يفحص النص في تمريرة واحدة ويختار بين مسار الفصحى الحافظ للإملاء (`FAST_MSA`)، أو مسار حفظ اللهجة وأدوات النفي (`DIALECT_PRESERVE`) لحماية كلمات مثل `مو، مب، مافيه، ماكو`، أو مسار تنقية ضجيج شبكات التواصل (`NOISY_SOCIAL`) الذي يضغط المدود والتطويل مع الحفاظ على الكلمات المشددة (`والللله ← والله`).
235
+ * **توفير التوكنات وتكاليف الـ APIs:** أظهرت التجارب المرجعية الموثقة على معجم `AraBERTv2` خفض إجمالي التوكنات بنسبة **46.57%** مقارنة بالنص الخام و**26.33%** مقارنة بمناهج التطبيع القديمة، مع استعادة **100%** من الكلمات المتأثرة بالتشكيل والكشيدة (`156 → 0 [UNK]`).
236
+
237
+ ---
238
+
239
+ <div align="center">
240
+
241
+ Authored and maintained by **Muneer Jarab Matrak Al-Theeb (منير جراب مترك الذيب)**
242
+ Released under the [MIT License](https://github.com/RealMtrx/misar/blob/main/LICENSE) © 2026
243
+
244
+ </div>
@@ -0,0 +1,62 @@
1
+ [build-system]
2
+ requires = ["setuptools>=68.0", "wheel"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "misar"
7
+ version = "1.0.0"
8
+ description = "Misar (مِسار): Adaptive context-aware Arabic NLP preprocessing router for optimizing Transformer language models."
9
+ readme = "README.md"
10
+ license = {text = "MIT"}
11
+ requires-python = ">=3.9"
12
+ authors = [
13
+ {name = "Muneer Jarab Matrak Al-Theeb"}
14
+ ]
15
+ keywords = [
16
+ "arabic",
17
+ "nlp",
18
+ "preprocessing",
19
+ "router",
20
+ "arabert",
21
+ "camelbert",
22
+ "llm",
23
+ "tokenization",
24
+ "dialect",
25
+ "saudi-arabia"
26
+ ]
27
+ classifiers = [
28
+ "Development Status :: 5 - Production/Stable",
29
+ "Intended Audience :: Developers",
30
+ "Intended Audience :: Science/Research",
31
+ "License :: OSI Approved :: MIT License",
32
+ "Natural Language :: Arabic",
33
+ "Operating System :: OS Independent",
34
+ "Programming Language :: Python :: 3",
35
+ "Programming Language :: Python :: 3.9",
36
+ "Programming Language :: Python :: 3.10",
37
+ "Programming Language :: Python :: 3.11",
38
+ "Programming Language :: Python :: 3.12",
39
+ "Programming Language :: Python :: 3.13",
40
+ "Programming Language :: Python :: 3.14",
41
+ "Topic :: Scientific/Engineering :: Artificial Intelligence",
42
+ "Topic :: Text Processing :: Linguistic",
43
+ ]
44
+ dependencies = []
45
+
46
+ [project.optional-dependencies]
47
+ huggingface = ["transformers>=4.35.0", "torch>=2.0.0"]
48
+ dev = ["pytest>=7.4.0", "build>=1.0.0", "twine>=5.0.0"]
49
+
50
+ [project.scripts]
51
+ misar = "misar.cli:main"
52
+
53
+ [project.urls]
54
+ Homepage = "https://github.com/RealMtrx/misar"
55
+ Repository = "https://github.com/RealMtrx/misar"
56
+ Issues = "https://github.com/RealMtrx/misar/issues"
57
+
58
+ [tool.setuptools.packages.find]
59
+ where = ["src"]
60
+
61
+ [tool.setuptools.package-data]
62
+ misar = ["web/*.html"]
misar-1.0.0/setup.cfg ADDED
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+
@@ -0,0 +1,36 @@
1
+ # -*- coding: utf-8 -*-
2
+ """
3
+ Misar (مِسار): Adaptive Context-Aware Arabic NLP Preprocessing Router.
4
+ """
5
+
6
+ from .pipelines import DialectPreservePipeline, MSALightPipeline, NoisySocialPipeline
7
+ from .profiler import LinguisticProfile, LinguisticProfiler, compute_shannon_entropy
8
+ from .router import MisarRouter, RoutingResult, preprocess, route
9
+ from .tokenizer import (
10
+ HeuristicSubwordTokenizer,
11
+ HuggingFaceTokenizerAdapter,
12
+ WordPieceTokenizer,
13
+ load_default_tokenizer,
14
+ )
15
+
16
+ __version__ = "1.0.0"
17
+ __author__ = "Muneer Jarab Matrak Al-Theeb"
18
+
19
+ __all__ = [
20
+ "__version__",
21
+ "__author__",
22
+ "MisarRouter",
23
+ "RoutingResult",
24
+ "LinguisticProfiler",
25
+ "LinguisticProfile",
26
+ "compute_shannon_entropy",
27
+ "MSALightPipeline",
28
+ "DialectPreservePipeline",
29
+ "NoisySocialPipeline",
30
+ "WordPieceTokenizer",
31
+ "HuggingFaceTokenizerAdapter",
32
+ "HeuristicSubwordTokenizer",
33
+ "load_default_tokenizer",
34
+ "route",
35
+ "preprocess",
36
+ ]
@@ -0,0 +1,5 @@
1
+ # -*- coding: utf-8 -*-
2
+ from .cli import main
3
+
4
+ if __name__ == "__main__":
5
+ main()