misar 2.0.0__tar.gz → 2.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {misar-2.0.0/src/misar.egg-info → misar-2.1.0}/PKG-INFO +48 -31
- {misar-2.0.0 → misar-2.1.0}/README.md +47 -30
- {misar-2.0.0 → misar-2.1.0}/pyproject.toml +1 -1
- {misar-2.0.0 → misar-2.1.0}/src/misar/__init__.py +8 -4
- {misar-2.0.0 → misar-2.1.0}/src/misar/cli.py +14 -0
- misar-2.1.0/src/misar/evaluation.py +144 -0
- {misar-2.0.0 → misar-2.1.0}/src/misar/lexicons.py +28 -3
- {misar-2.0.0 → misar-2.1.0}/src/misar/pipelines.py +16 -5
- {misar-2.0.0 → misar-2.1.0}/src/misar/profiler.py +5 -1
- {misar-2.0.0 → misar-2.1.0}/src/misar/router.py +130 -7
- {misar-2.0.0 → misar-2.1.0}/src/misar/web/index.html +183 -21
- {misar-2.0.0 → misar-2.1.0}/src/misar/web/server.py +90 -28
- {misar-2.0.0 → misar-2.1.0/src/misar.egg-info}/PKG-INFO +48 -31
- {misar-2.0.0 → misar-2.1.0}/src/misar.egg-info/SOURCES.txt +1 -0
- {misar-2.0.0 → misar-2.1.0}/tests/test_misar.py +105 -0
- {misar-2.0.0 → misar-2.1.0}/LICENSE +0 -0
- {misar-2.0.0 → misar-2.1.0}/setup.cfg +0 -0
- {misar-2.0.0 → misar-2.1.0}/src/misar/__main__.py +0 -0
- {misar-2.0.0 → misar-2.1.0}/src/misar/benchmark.py +0 -0
- {misar-2.0.0 → misar-2.1.0}/src/misar/tokenizer.py +0 -0
- {misar-2.0.0 → misar-2.1.0}/src/misar/web/__init__.py +0 -0
- {misar-2.0.0 → misar-2.1.0}/src/misar.egg-info/dependency_links.txt +0 -0
- {misar-2.0.0 → misar-2.1.0}/src/misar.egg-info/entry_points.txt +0 -0
- {misar-2.0.0 → misar-2.1.0}/src/misar.egg-info/requires.txt +0 -0
- {misar-2.0.0 → misar-2.1.0}/src/misar.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: misar
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.1.0
|
|
4
4
|
Summary: Misar (مِسار): Adaptive context-aware Arabic NLP preprocessing router for optimizing Transformer language models.
|
|
5
5
|
Author: Muneer Jarab Matrak Al-Theeb
|
|
6
6
|
License: MIT
|
|
@@ -47,9 +47,10 @@ Dynamic: license-file
|
|
|
47
47
|
[](https://www.python.org/downloads/)
|
|
48
48
|
[](#installation)
|
|
49
49
|
[](https://github.com/RealMtrx/misar/blob/main/LICENSE)
|
|
50
|
-
[](#empirical-benchmarks)
|
|
51
51
|
[](#empirical-benchmarks)
|
|
52
|
-
[%20attention%20saved--71.8%25-7c3aed.svg)](#scientific-cross-model-evaluation)
|
|
53
|
+
[](#empirical-benchmarks)
|
|
53
54
|
|
|
54
55
|
[Architecture](#architecture) • [Benchmarks](#empirical-benchmarks) • [Installation](#installation) • [Quick Start](#quick-start) • [CLI & Workbench](#cli--web-workbench) • [Citation](#citation) • [النسخة العربية](#-نظرة-عامة-بالعربية)
|
|
55
56
|
|
|
@@ -71,16 +72,16 @@ Modern Arabic NLP deployments face a structural bottleneck: forcing heterogeneou
|
|
|
71
72
|
|
|
72
73
|
**Misar (`مِسار`)** solves these bottlenecks **without retraining any model (`Zero Parameter Update`)** by acting as an ultra-lightweight, zero-dependency, linear-time $O(n)$ routing gateway directly in front of **AraBERTv2, MARBERTv2, CAMeLBERT-Mix, ARBERTv2, QARiB**, or generative LLMs.
|
|
73
74
|
|
|
74
|
-
### How Misar v2.
|
|
75
|
+
### How Misar v2.1 Compares to Existing Arabic Preprocessors
|
|
75
76
|
|
|
76
|
-
| Capability | `ArabertPreprocessor` | `tnkeeh` | `CAMeL Tools` | **Misar v2.
|
|
77
|
+
| Capability | `ArabertPreprocessor` | `tnkeeh` | `CAMeL Tools` | **Misar v2.1 (`مِسار`)** |
|
|
77
78
|
| :--- | :---: | :---: | :---: | :---: |
|
|
78
|
-
| **Dynamic $O(n)$
|
|
79
|
+
| **Dynamic $O(n)$ + Clause-Aware Routing** | Static pipeline | Static pipeline | Manual utilities | **Per-text & Multi-Clause `route_document()`** |
|
|
79
80
|
| **Zero Mandatory Dependencies** | Requires `pyarabic` / extras | Light deps | Heavy ML / data packs | **100% Pure Python Stdlib** |
|
|
80
|
-
| **
|
|
81
|
+
| **Unicode Presentation Forms & Farsi Lookalikes** | Partial regex | Partial regex | Partial utils | **`NFKC` + Lookalike Defense (`ﺍﻟﺒﺮﻣﺠﺔ/کود → 0 [UNK]`)** |
|
|
82
|
+
| **Vocab-Verified Auto-Heal & Semantic Pairs** | Collapses `ة→ه` (`0%` pair disambig) | Optional regex | Separate disambig | **`100%` Minimal-Pair Disambiguation + Auto-Heal** |
|
|
81
83
|
| **5 Arabic Transformer Vocabs (`do_lower_case`)** | AraBERT focus | Generic | General NLP | **AraBERTv2, MARBERTv2, ARBERTv2, CAMeLBERT, QARiB** |
|
|
82
|
-
| **
|
|
83
|
-
| **3-Way Scientific Web Workbench + REST API** | — | — | CLI only | **Built-in Live Comparator UI (`Raw` vs `Legacy` vs `Misar`)** |
|
|
84
|
+
| **Scientific Evaluator & $O(L^2)$ Telemetry** | — | — | CLI only | **Built-in `misar eval --all-models` & Live UI** |
|
|
84
85
|
|
|
85
86
|
---
|
|
86
87
|
|
|
@@ -94,7 +95,7 @@ flowchart LR
|
|
|
94
95
|
B -->|"High Noise / Tatweel / Stretches ≥ 3"| C["NOISY_SOCIAL (+ Hybrid Dialect)<br/>Gemination-Safe Collapse + Emoji Polarity"]
|
|
95
96
|
B -->|"Dialect Markers / Protected Negations"| D["DIALECT_PRESERVE<br/>Negation Protection + Vocab Auto-Heal"]
|
|
96
97
|
B -->|"Clean MSA / Classical Diacritized"| E["FAST_MSA<br/>Strip Harakat · Preserve أ/إ/آ, ة, ى"]
|
|
97
|
-
C --> F["AraBERTv2 / MARBERTv2 / CAMeLBERT / LLMs<br/>(-46.91% Tokens ·
|
|
98
|
+
C --> F["AraBERTv2 / MARBERTv2 / CAMeLBERT / LLMs<br/>(-46.91% Tokens · -71.8% O(L²) Attn · 4,771 texts/s)"]
|
|
98
99
|
D --> F
|
|
99
100
|
E --> F
|
|
100
101
|
```
|
|
@@ -111,9 +112,21 @@ Measured reproducibly via `python benchmarks/run_benchmark.py` against the authe
|
|
|
111
112
|
| **Classical Diacritized (Heritage)** | 3 | 186 *(147 OOV `[UNK]`)* | 48 | **39** | **`-79.03%`** vs Raw · **`-18.75%`** vs Legacy (`0 [UNK]`) |
|
|
112
113
|
| **Saudi / Gulf / Levantine / Maghrebi Dev** | 6 | 124 | 131 | **117** | **`-5.65%`** vs Raw · **`-10.69%`** vs Legacy (+ Auto-Heal & Negations) |
|
|
113
114
|
| **Noisy Tech Social & Code-Switching** | 3 | 102 | 40 | **36** | **`-64.71%`** vs Raw · **`-10.00%`** vs Legacy + gemination intact |
|
|
114
|
-
| **Full Corpus Summary** | **15** | **469** | **298** | **249** | **`-46.91%` vs Raw · `-16.44%` vs Fair Legacy · `4,
|
|
115
|
+
| **Full Corpus Summary** | **15** | **469** | **298** | **249** | **`-46.91%` vs Raw · `-16.44%` vs Fair Legacy · `4,771 texts/s`** |
|
|
115
116
|
|
|
116
|
-
> **Key Scientific Proof:** Even when competing against an identical full-denoising Legacy Static baseline (`URL/Emoji/Hashtag/Stretch removal + ة→ه, أ→ا`), Misar v2.
|
|
117
|
+
> **Key Scientific Proof:** Even when competing against an identical full-denoising Legacy Static baseline (`URL/Emoji/Hashtag/Stretch removal + ة→ه, أ→ا`), Misar v2.1 still saves **16.44%** fewer tokens than Legacy Static (**46.91%** vs Raw), eliminates **`157 → 0 [UNK]`** tokens, repairs broken orthography (`Auto-Heal`), preserves dialectal negations, and streams at **4,771 texts/sec** on CPU.
|
|
118
|
+
|
|
119
|
+
### Scientific Cross-Model Evaluation (`misar eval --all-models`)
|
|
120
|
+
|
|
121
|
+
Measured live across all 5 authentic Hugging Face Arabic Transformer vocabularies (`evaluation.py`):
|
|
122
|
+
|
|
123
|
+
| Model Key | Vocabulary Backend | Semantic Minimal-Pair Disambig (`Misar` vs `Legacy`) | Dialect Negation Safety | Emoji Polarity Retention | Tokens vs Raw / vs Legacy | Quadratic $O(L^2)$ Attention Saved |
|
|
124
|
+
| :--- | :--- | :---: | :---: | :---: | :---: | :---: |
|
|
125
|
+
| **`arabertv02`** | `aubmindlab/bert-base-arabertv02` | **`100%` vs `0%`** | **`100%`** | **`100%`** | **`-46.9%` / `-16.4%`** | **`-71.8%`** |
|
|
126
|
+
| **`arbertv2`** | `UBC-NLP/ARBERTv2` | **`100%` vs `0%`** | **`100%`** | **`100%`** | **`-49.3%` / `-15.6%`** | **`-74.3%`** |
|
|
127
|
+
| **`camelbert-mix`** | `CAMeL-Lab/bert-base-arabic-camelbert-mix` | **`100%` vs `0%`** | **`100%`** | **`100%`** | **`-43.2%` / `-8.7%`** | **`-67.8%`** |
|
|
128
|
+
| **`marbertv2`** | `UBC-NLP/MARBERTv2` | **`100%` vs `0%`** | **`100%`** | **`100%`** | **`-47.9%` / `-7.5%`** | **`-72.8%`** |
|
|
129
|
+
| **`qarib`** | `qarib/bert-base-qarib` | **`100%` vs `0%`** | **`100%`** | **`100%`** | **`-45.5%` / `-9.0%`** | **`-70.3%`** |
|
|
117
130
|
|
|
118
131
|
---
|
|
119
132
|
|
|
@@ -158,23 +171,24 @@ print("Reduction: ", f"-{result.token_reduction_pct:.1f}%")
|
|
|
158
171
|
print("Latency: ", f"{result.latency_ms:.3f} ms")
|
|
159
172
|
```
|
|
160
173
|
|
|
161
|
-
### 2.
|
|
174
|
+
### 2. Clause-Aware Multi-Route Documents, Unicode PDF Defense & Batching
|
|
162
175
|
|
|
163
176
|
```python
|
|
164
177
|
from misar import MisarRouter, preprocess, preprocess_batch
|
|
165
178
|
|
|
166
|
-
|
|
167
|
-
clean = preprocess("الســــلام علــــيكم ومافيه مشكلة #تطوير_البرمجيات")
|
|
168
|
-
# -> "السلام عليكم ومافيه مشكلة تطوير البرمجيات"
|
|
179
|
+
router = MisarRouter(model_key="arabertv02", emoji_mode="sentiment", auto_heal=True)
|
|
169
180
|
|
|
170
|
-
#
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
181
|
+
# Recovers PDF Arabic Presentation Forms + Persian keyboard lookalikes (کود -> كود)
|
|
182
|
+
# AND routes formal vs. social sentences independently in multi-clause documents:
|
|
183
|
+
doc_res = router.route_document(
|
|
184
|
+
"ﺍﻟﺒﺮﻣﺠﺔ هندسة دقيقة باستخدام کود سريع.\n"
|
|
185
|
+
"السيرفررررر الحين يجنن في الـ API والله يستاهل 😍🚀"
|
|
186
|
+
)
|
|
187
|
+
print("Distribution:", doc_res.route_distribution) # {'FAST_MSA': 1, 'DIALECT_PRESERVE': 0, 'NOISY_SOCIAL': 1}
|
|
188
|
+
print("Attention Saved:", f"-{doc_res.attention_savings_pct:.1f}%")
|
|
175
189
|
|
|
176
190
|
# Fast streaming batch preprocessing & Hugging Face Datasets .map(..., batched=True)
|
|
177
|
-
cleaned_list =
|
|
191
|
+
cleaned_list = preprocess_batch(["نص أول", "نص ثاااني 😡"])
|
|
178
192
|
# dataset = dataset.map(router.hf_dataset_map, batched=True)
|
|
179
193
|
```
|
|
180
194
|
|
|
@@ -206,8 +220,9 @@ misar route "ياخي البرمجه في المنصه هاذي بطيييئة
|
|
|
206
220
|
# Select target model vocab (arabertv02, camelbert-mix, marbertv2, arbertv2, qarib)
|
|
207
221
|
misar route "أعلنت منصة المطورين عن ترقية واجهة البرمجة API" --model marbertv2 --json
|
|
208
222
|
|
|
209
|
-
# Run the reproducible 15-sample benchmark suite
|
|
223
|
+
# Run the reproducible 15-sample benchmark suite & 5-model scientific evaluation
|
|
210
224
|
misar bench
|
|
225
|
+
misar eval --all-models
|
|
211
226
|
|
|
212
227
|
# Launch the interactive RTL Web Workbench & hardened REST API on http://127.0.0.1:8050
|
|
213
228
|
misar serve
|
|
@@ -219,7 +234,8 @@ misar serve
|
|
|
219
234
|
| :--- | :--- | :--- |
|
|
220
235
|
| `GET` | `/api/health` | Service status, version, and active tokenizer backend |
|
|
221
236
|
| `GET` | `/api/suite` | Evaluates the 6 standard cross-domain reference samples |
|
|
222
|
-
| `POST` | `/api/route` | Body `{"text": "..."}` →
|
|
237
|
+
| `POST` | `/api/route` | Body `{"text": "..."}` → Full `RoutingResult` (supports `document_mode: true`) |
|
|
238
|
+
| `POST` | `/api/document` | Body `{"text": "..."}` → Clause-aware `DocumentRoutingResult` + $O(L^2)$ metrics |
|
|
223
239
|
| `POST` | `/api/batch` | Body `{"texts": ["...", "..."]}` → Batch routes up to 500 strings (`1 MiB` payload limit) |
|
|
224
240
|
|
|
225
241
|
---
|
|
@@ -240,12 +256,13 @@ misar/
|
|
|
240
256
|
│ ├── pipelines.py # FAST_MSA, DIALECT_PRESERVE & NOISY_SOCIAL passes
|
|
241
257
|
│ ├── tokenizer.py # Pure-Python WordPieceTokenizer + HF & offline adapters
|
|
242
258
|
│ ├── router.py # MisarRouter orchestration & wrap_hf_pipeline
|
|
259
|
+
│ ├── evaluation.py # Multi-model scientific evaluation suite (disambig/polarity/O(L^2))
|
|
243
260
|
│ ├── benchmark.py # Shared benchmark suite module
|
|
244
|
-
│ ├── cli.py # `misar` CLI (`demo`, `route`, `bench`, `serve`)
|
|
261
|
+
│ ├── cli.py # `misar` CLI (`demo`, `route`, `bench`, `eval`, `serve`)
|
|
245
262
|
│ └── web/
|
|
246
263
|
│ ├── server.py # Hardened zero-dependency ThreadingHTTPServer REST API
|
|
247
264
|
│ └── index.html # Interactive Arabic RTL Workbench & token visualizer
|
|
248
|
-
├── tests/test_misar.py #
|
|
265
|
+
├── tests/test_misar.py # 39 unit, v2.1 feature, server & real-vocab tests
|
|
249
266
|
├── CITATION.cff # Native GitHub repository citation metadata
|
|
250
267
|
├── CONTRIBUTING.md # Developer setup & contribution standards
|
|
251
268
|
├── SECURITY.md # Built-in API & frontend hardening controls
|
|
@@ -262,7 +279,7 @@ misar/
|
|
|
262
279
|
python -m unittest discover -s tests -v
|
|
263
280
|
```
|
|
264
281
|
|
|
265
|
-
All **
|
|
282
|
+
All **39 automated tests** verify profiler mathematics, Unicode Presentation Form & Farsi lookalike recovery (`ﺍﻟﺒﺮﻣﺠﺔ/کود → 0 [UNK]`), long-essay social artifact sanitization, Clause-Aware `route_document()` multi-clause dispatch, MSA homograph resistance, Saudi/Gulf/Levantine/Maghrebi negation preservation, gemination protection (`والللله → والله`), pronoun-safe Vocab Auto-Heal (`البرمجه → البرمجة` while protecting `بيته/ملفه/مديره`), clitic code-switching (`الـ Backend → الBackend`), `do_lower_case` parity for `MARBERTv2`/`ARBERTv2`, opt-in Arabizi tech safety, scientific minimal-pair evaluation (`100% vs 0%`), fast batch/HF dataset mapping, and hardened REST server HTTP security bounds.
|
|
266
283
|
|
|
267
284
|
---
|
|
268
285
|
|
|
@@ -275,7 +292,7 @@ If you use **Misar** in academic research, benchmarks, or commercial NLP pipelin
|
|
|
275
292
|
author = {Muneer Jarab Matrak Al-Theeb},
|
|
276
293
|
title = {Misar (مِسار): Adaptive Context-Aware Arabic NLP Preprocessing Router for Transformer Language Models},
|
|
277
294
|
year = {2026},
|
|
278
|
-
version = {2.
|
|
295
|
+
version = {2.1.0},
|
|
279
296
|
url = {https://github.com/RealMtrx/misar},
|
|
280
297
|
license = {MIT}
|
|
281
298
|
}
|
|
@@ -285,10 +302,10 @@ If you use **Misar** in academic research, benchmarks, or commercial NLP pipelin
|
|
|
285
302
|
|
|
286
303
|
## 📖 نظرة عامة بالعربية
|
|
287
304
|
|
|
288
|
-
**مِسار (`Misar v2.
|
|
305
|
+
**مِسار (`Misar v2.1.0`)** هي مكتبة وموجّه بايثون مفتوح المصدر وخفيف الوزن (بلا أي تبعيات خارجية إجبارية) صُمِّم ليعمل كطبقة بوابة ذكية في زمن خطي $O(n)$ (**~0.27 مللي ثانية** للنص وبسرعة **4,771 نص/ثانية** في المعالجة الدفعية) قبل نماذج المحولات العربية الخمسة (`AraBERTv2`, `MARBERTv2`, `CAMeLBERT`, `ARBERTv2`, `QARiB`) والنماذج التوليدية (`ALLaM` و `LLMs`) **دون الحاجة إلى إعادة تدريب النموذج (`Zero Parameter Update`)**.
|
|
289
306
|
|
|
290
|
-
* **توجيه
|
|
291
|
-
* **توفير التوكنات وتكاليف
|
|
307
|
+
* **توجيه سياقي متعدد الجمل (`route_document`) + دفاع Unicode:** يختار ديناميكياً على مستوى المستند أو كل فقرة وجملة بين `FAST_MSA` و`DIALECT_PRESERVE` و`NOISY_SOCIAL`، يستعيد أشكال العرض المعزولة لملفات PDF (`NFKC`: `ﺍﻟﺒﺮﻣﺠﺔ ← البرمجة`) ومحارف لوحات المفاتيح الفارسية/الأردية (`ک/ی ← ك/ي`)، ويرمّم أخطاء التاء المربوطة (`البرمجه ← البرمجة`) دون المساس بالضمائر (`بيته، ملفه، مديره`).
|
|
308
|
+
* **توفير التوكنات وتكاليف الانتباه التربيعي $O(L^2)$:** أظهرت التجارب الموثقة (`misar eval --all-models`) تفوق **100% مقابل 0%** في تمييز الثنائيات الصرفية (`عالمة/عالمه`)، وحفظ **100%** من النفي اللهجي وقطبية المشاعر، وخفض التوكنات بنسبة **46.91%** مقابل الخام و**16.44%** مقابل المناهج التقليدية (ما يوفّر **67.8%–74.3%** من عمليات الانتباه $O(L^2)$ واستعادة `157 → 0 [UNK]`).
|
|
292
309
|
|
|
293
310
|
---
|
|
294
311
|
|
|
@@ -10,9 +10,10 @@
|
|
|
10
10
|
[](https://www.python.org/downloads/)
|
|
11
11
|
[](#installation)
|
|
12
12
|
[](https://github.com/RealMtrx/misar/blob/main/LICENSE)
|
|
13
|
-
[](#empirical-benchmarks)
|
|
14
14
|
[](#empirical-benchmarks)
|
|
15
|
-
[%20attention%20saved--71.8%25-7c3aed.svg)](#scientific-cross-model-evaluation)
|
|
16
|
+
[](#empirical-benchmarks)
|
|
16
17
|
|
|
17
18
|
[Architecture](#architecture) • [Benchmarks](#empirical-benchmarks) • [Installation](#installation) • [Quick Start](#quick-start) • [CLI & Workbench](#cli--web-workbench) • [Citation](#citation) • [النسخة العربية](#-نظرة-عامة-بالعربية)
|
|
18
19
|
|
|
@@ -34,16 +35,16 @@ Modern Arabic NLP deployments face a structural bottleneck: forcing heterogeneou
|
|
|
34
35
|
|
|
35
36
|
**Misar (`مِسار`)** solves these bottlenecks **without retraining any model (`Zero Parameter Update`)** by acting as an ultra-lightweight, zero-dependency, linear-time $O(n)$ routing gateway directly in front of **AraBERTv2, MARBERTv2, CAMeLBERT-Mix, ARBERTv2, QARiB**, or generative LLMs.
|
|
36
37
|
|
|
37
|
-
### How Misar v2.
|
|
38
|
+
### How Misar v2.1 Compares to Existing Arabic Preprocessors
|
|
38
39
|
|
|
39
|
-
| Capability | `ArabertPreprocessor` | `tnkeeh` | `CAMeL Tools` | **Misar v2.
|
|
40
|
+
| Capability | `ArabertPreprocessor` | `tnkeeh` | `CAMeL Tools` | **Misar v2.1 (`مِسار`)** |
|
|
40
41
|
| :--- | :---: | :---: | :---: | :---: |
|
|
41
|
-
| **Dynamic $O(n)$
|
|
42
|
+
| **Dynamic $O(n)$ + Clause-Aware Routing** | Static pipeline | Static pipeline | Manual utilities | **Per-text & Multi-Clause `route_document()`** |
|
|
42
43
|
| **Zero Mandatory Dependencies** | Requires `pyarabic` / extras | Light deps | Heavy ML / data packs | **100% Pure Python Stdlib** |
|
|
43
|
-
| **
|
|
44
|
+
| **Unicode Presentation Forms & Farsi Lookalikes** | Partial regex | Partial regex | Partial utils | **`NFKC` + Lookalike Defense (`ﺍﻟﺒﺮﻣﺠﺔ/کود → 0 [UNK]`)** |
|
|
45
|
+
| **Vocab-Verified Auto-Heal & Semantic Pairs** | Collapses `ة→ه` (`0%` pair disambig) | Optional regex | Separate disambig | **`100%` Minimal-Pair Disambiguation + Auto-Heal** |
|
|
44
46
|
| **5 Arabic Transformer Vocabs (`do_lower_case`)** | AraBERT focus | Generic | General NLP | **AraBERTv2, MARBERTv2, ARBERTv2, CAMeLBERT, QARiB** |
|
|
45
|
-
| **
|
|
46
|
-
| **3-Way Scientific Web Workbench + REST API** | — | — | CLI only | **Built-in Live Comparator UI (`Raw` vs `Legacy` vs `Misar`)** |
|
|
47
|
+
| **Scientific Evaluator & $O(L^2)$ Telemetry** | — | — | CLI only | **Built-in `misar eval --all-models` & Live UI** |
|
|
47
48
|
|
|
48
49
|
---
|
|
49
50
|
|
|
@@ -57,7 +58,7 @@ flowchart LR
|
|
|
57
58
|
B -->|"High Noise / Tatweel / Stretches ≥ 3"| C["NOISY_SOCIAL (+ Hybrid Dialect)<br/>Gemination-Safe Collapse + Emoji Polarity"]
|
|
58
59
|
B -->|"Dialect Markers / Protected Negations"| D["DIALECT_PRESERVE<br/>Negation Protection + Vocab Auto-Heal"]
|
|
59
60
|
B -->|"Clean MSA / Classical Diacritized"| E["FAST_MSA<br/>Strip Harakat · Preserve أ/إ/آ, ة, ى"]
|
|
60
|
-
C --> F["AraBERTv2 / MARBERTv2 / CAMeLBERT / LLMs<br/>(-46.91% Tokens ·
|
|
61
|
+
C --> F["AraBERTv2 / MARBERTv2 / CAMeLBERT / LLMs<br/>(-46.91% Tokens · -71.8% O(L²) Attn · 4,771 texts/s)"]
|
|
61
62
|
D --> F
|
|
62
63
|
E --> F
|
|
63
64
|
```
|
|
@@ -74,9 +75,21 @@ Measured reproducibly via `python benchmarks/run_benchmark.py` against the authe
|
|
|
74
75
|
| **Classical Diacritized (Heritage)** | 3 | 186 *(147 OOV `[UNK]`)* | 48 | **39** | **`-79.03%`** vs Raw · **`-18.75%`** vs Legacy (`0 [UNK]`) |
|
|
75
76
|
| **Saudi / Gulf / Levantine / Maghrebi Dev** | 6 | 124 | 131 | **117** | **`-5.65%`** vs Raw · **`-10.69%`** vs Legacy (+ Auto-Heal & Negations) |
|
|
76
77
|
| **Noisy Tech Social & Code-Switching** | 3 | 102 | 40 | **36** | **`-64.71%`** vs Raw · **`-10.00%`** vs Legacy + gemination intact |
|
|
77
|
-
| **Full Corpus Summary** | **15** | **469** | **298** | **249** | **`-46.91%` vs Raw · `-16.44%` vs Fair Legacy · `4,
|
|
78
|
+
| **Full Corpus Summary** | **15** | **469** | **298** | **249** | **`-46.91%` vs Raw · `-16.44%` vs Fair Legacy · `4,771 texts/s`** |
|
|
78
79
|
|
|
79
|
-
> **Key Scientific Proof:** Even when competing against an identical full-denoising Legacy Static baseline (`URL/Emoji/Hashtag/Stretch removal + ة→ه, أ→ا`), Misar v2.
|
|
80
|
+
> **Key Scientific Proof:** Even when competing against an identical full-denoising Legacy Static baseline (`URL/Emoji/Hashtag/Stretch removal + ة→ه, أ→ا`), Misar v2.1 still saves **16.44%** fewer tokens than Legacy Static (**46.91%** vs Raw), eliminates **`157 → 0 [UNK]`** tokens, repairs broken orthography (`Auto-Heal`), preserves dialectal negations, and streams at **4,771 texts/sec** on CPU.
|
|
81
|
+
|
|
82
|
+
### Scientific Cross-Model Evaluation (`misar eval --all-models`)
|
|
83
|
+
|
|
84
|
+
Measured live across all 5 authentic Hugging Face Arabic Transformer vocabularies (`evaluation.py`):
|
|
85
|
+
|
|
86
|
+
| Model Key | Vocabulary Backend | Semantic Minimal-Pair Disambig (`Misar` vs `Legacy`) | Dialect Negation Safety | Emoji Polarity Retention | Tokens vs Raw / vs Legacy | Quadratic $O(L^2)$ Attention Saved |
|
|
87
|
+
| :--- | :--- | :---: | :---: | :---: | :---: | :---: |
|
|
88
|
+
| **`arabertv02`** | `aubmindlab/bert-base-arabertv02` | **`100%` vs `0%`** | **`100%`** | **`100%`** | **`-46.9%` / `-16.4%`** | **`-71.8%`** |
|
|
89
|
+
| **`arbertv2`** | `UBC-NLP/ARBERTv2` | **`100%` vs `0%`** | **`100%`** | **`100%`** | **`-49.3%` / `-15.6%`** | **`-74.3%`** |
|
|
90
|
+
| **`camelbert-mix`** | `CAMeL-Lab/bert-base-arabic-camelbert-mix` | **`100%` vs `0%`** | **`100%`** | **`100%`** | **`-43.2%` / `-8.7%`** | **`-67.8%`** |
|
|
91
|
+
| **`marbertv2`** | `UBC-NLP/MARBERTv2` | **`100%` vs `0%`** | **`100%`** | **`100%`** | **`-47.9%` / `-7.5%`** | **`-72.8%`** |
|
|
92
|
+
| **`qarib`** | `qarib/bert-base-qarib` | **`100%` vs `0%`** | **`100%`** | **`100%`** | **`-45.5%` / `-9.0%`** | **`-70.3%`** |
|
|
80
93
|
|
|
81
94
|
---
|
|
82
95
|
|
|
@@ -121,23 +134,24 @@ print("Reduction: ", f"-{result.token_reduction_pct:.1f}%")
|
|
|
121
134
|
print("Latency: ", f"{result.latency_ms:.3f} ms")
|
|
122
135
|
```
|
|
123
136
|
|
|
124
|
-
### 2.
|
|
137
|
+
### 2. Clause-Aware Multi-Route Documents, Unicode PDF Defense & Batching
|
|
125
138
|
|
|
126
139
|
```python
|
|
127
140
|
from misar import MisarRouter, preprocess, preprocess_batch
|
|
128
141
|
|
|
129
|
-
|
|
130
|
-
clean = preprocess("الســــلام علــــيكم ومافيه مشكلة #تطوير_البرمجيات")
|
|
131
|
-
# -> "السلام عليكم ومافيه مشكلة تطوير البرمجيات"
|
|
142
|
+
router = MisarRouter(model_key="arabertv02", emoji_mode="sentiment", auto_heal=True)
|
|
132
143
|
|
|
133
|
-
#
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
144
|
+
# Recovers PDF Arabic Presentation Forms + Persian keyboard lookalikes (کود -> كود)
|
|
145
|
+
# AND routes formal vs. social sentences independently in multi-clause documents:
|
|
146
|
+
doc_res = router.route_document(
|
|
147
|
+
"ﺍﻟﺒﺮﻣﺠﺔ هندسة دقيقة باستخدام کود سريع.\n"
|
|
148
|
+
"السيرفررررر الحين يجنن في الـ API والله يستاهل 😍🚀"
|
|
149
|
+
)
|
|
150
|
+
print("Distribution:", doc_res.route_distribution) # {'FAST_MSA': 1, 'DIALECT_PRESERVE': 0, 'NOISY_SOCIAL': 1}
|
|
151
|
+
print("Attention Saved:", f"-{doc_res.attention_savings_pct:.1f}%")
|
|
138
152
|
|
|
139
153
|
# Fast streaming batch preprocessing & Hugging Face Datasets .map(..., batched=True)
|
|
140
|
-
cleaned_list =
|
|
154
|
+
cleaned_list = preprocess_batch(["نص أول", "نص ثاااني 😡"])
|
|
141
155
|
# dataset = dataset.map(router.hf_dataset_map, batched=True)
|
|
142
156
|
```
|
|
143
157
|
|
|
@@ -169,8 +183,9 @@ misar route "ياخي البرمجه في المنصه هاذي بطيييئة
|
|
|
169
183
|
# Select target model vocab (arabertv02, camelbert-mix, marbertv2, arbertv2, qarib)
|
|
170
184
|
misar route "أعلنت منصة المطورين عن ترقية واجهة البرمجة API" --model marbertv2 --json
|
|
171
185
|
|
|
172
|
-
# Run the reproducible 15-sample benchmark suite
|
|
186
|
+
# Run the reproducible 15-sample benchmark suite & 5-model scientific evaluation
|
|
173
187
|
misar bench
|
|
188
|
+
misar eval --all-models
|
|
174
189
|
|
|
175
190
|
# Launch the interactive RTL Web Workbench & hardened REST API on http://127.0.0.1:8050
|
|
176
191
|
misar serve
|
|
@@ -182,7 +197,8 @@ misar serve
|
|
|
182
197
|
| :--- | :--- | :--- |
|
|
183
198
|
| `GET` | `/api/health` | Service status, version, and active tokenizer backend |
|
|
184
199
|
| `GET` | `/api/suite` | Evaluates the 6 standard cross-domain reference samples |
|
|
185
|
-
| `POST` | `/api/route` | Body `{"text": "..."}` →
|
|
200
|
+
| `POST` | `/api/route` | Body `{"text": "..."}` → Full `RoutingResult` (supports `document_mode: true`) |
|
|
201
|
+
| `POST` | `/api/document` | Body `{"text": "..."}` → Clause-aware `DocumentRoutingResult` + $O(L^2)$ metrics |
|
|
186
202
|
| `POST` | `/api/batch` | Body `{"texts": ["...", "..."]}` → Batch routes up to 500 strings (`1 MiB` payload limit) |
|
|
187
203
|
|
|
188
204
|
---
|
|
@@ -203,12 +219,13 @@ misar/
|
|
|
203
219
|
│ ├── pipelines.py # FAST_MSA, DIALECT_PRESERVE & NOISY_SOCIAL passes
|
|
204
220
|
│ ├── tokenizer.py # Pure-Python WordPieceTokenizer + HF & offline adapters
|
|
205
221
|
│ ├── router.py # MisarRouter orchestration & wrap_hf_pipeline
|
|
222
|
+
│ ├── evaluation.py # Multi-model scientific evaluation suite (disambig/polarity/O(L^2))
|
|
206
223
|
│ ├── benchmark.py # Shared benchmark suite module
|
|
207
|
-
│ ├── cli.py # `misar` CLI (`demo`, `route`, `bench`, `serve`)
|
|
224
|
+
│ ├── cli.py # `misar` CLI (`demo`, `route`, `bench`, `eval`, `serve`)
|
|
208
225
|
│ └── web/
|
|
209
226
|
│ ├── server.py # Hardened zero-dependency ThreadingHTTPServer REST API
|
|
210
227
|
│ └── index.html # Interactive Arabic RTL Workbench & token visualizer
|
|
211
|
-
├── tests/test_misar.py #
|
|
228
|
+
├── tests/test_misar.py # 39 unit, v2.1 feature, server & real-vocab tests
|
|
212
229
|
├── CITATION.cff # Native GitHub repository citation metadata
|
|
213
230
|
├── CONTRIBUTING.md # Developer setup & contribution standards
|
|
214
231
|
├── SECURITY.md # Built-in API & frontend hardening controls
|
|
@@ -225,7 +242,7 @@ misar/
|
|
|
225
242
|
python -m unittest discover -s tests -v
|
|
226
243
|
```
|
|
227
244
|
|
|
228
|
-
All **
|
|
245
|
+
All **39 automated tests** verify profiler mathematics, Unicode Presentation Form & Farsi lookalike recovery (`ﺍﻟﺒﺮﻣﺠﺔ/کود → 0 [UNK]`), long-essay social artifact sanitization, Clause-Aware `route_document()` multi-clause dispatch, MSA homograph resistance, Saudi/Gulf/Levantine/Maghrebi negation preservation, gemination protection (`والللله → والله`), pronoun-safe Vocab Auto-Heal (`البرمجه → البرمجة` while protecting `بيته/ملفه/مديره`), clitic code-switching (`الـ Backend → الBackend`), `do_lower_case` parity for `MARBERTv2`/`ARBERTv2`, opt-in Arabizi tech safety, scientific minimal-pair evaluation (`100% vs 0%`), fast batch/HF dataset mapping, and hardened REST server HTTP security bounds.
|
|
229
246
|
|
|
230
247
|
---
|
|
231
248
|
|
|
@@ -238,7 +255,7 @@ If you use **Misar** in academic research, benchmarks, or commercial NLP pipelin
|
|
|
238
255
|
author = {Muneer Jarab Matrak Al-Theeb},
|
|
239
256
|
title = {Misar (مِسار): Adaptive Context-Aware Arabic NLP Preprocessing Router for Transformer Language Models},
|
|
240
257
|
year = {2026},
|
|
241
|
-
version = {2.
|
|
258
|
+
version = {2.1.0},
|
|
242
259
|
url = {https://github.com/RealMtrx/misar},
|
|
243
260
|
license = {MIT}
|
|
244
261
|
}
|
|
@@ -248,10 +265,10 @@ If you use **Misar** in academic research, benchmarks, or commercial NLP pipelin
|
|
|
248
265
|
|
|
249
266
|
## 📖 نظرة عامة بالعربية
|
|
250
267
|
|
|
251
|
-
**مِسار (`Misar v2.
|
|
268
|
+
**مِسار (`Misar v2.1.0`)** هي مكتبة وموجّه بايثون مفتوح المصدر وخفيف الوزن (بلا أي تبعيات خارجية إجبارية) صُمِّم ليعمل كطبقة بوابة ذكية في زمن خطي $O(n)$ (**~0.27 مللي ثانية** للنص وبسرعة **4,771 نص/ثانية** في المعالجة الدفعية) قبل نماذج المحولات العربية الخمسة (`AraBERTv2`, `MARBERTv2`, `CAMeLBERT`, `ARBERTv2`, `QARiB`) والنماذج التوليدية (`ALLaM` و `LLMs`) **دون الحاجة إلى إعادة تدريب النموذج (`Zero Parameter Update`)**.
|
|
252
269
|
|
|
253
|
-
* **توجيه
|
|
254
|
-
* **توفير التوكنات وتكاليف
|
|
270
|
+
* **توجيه سياقي متعدد الجمل (`route_document`) + دفاع Unicode:** يختار ديناميكياً على مستوى المستند أو كل فقرة وجملة بين `FAST_MSA` و`DIALECT_PRESERVE` و`NOISY_SOCIAL`، يستعيد أشكال العرض المعزولة لملفات PDF (`NFKC`: `ﺍﻟﺒﺮﻣﺠﺔ ← البرمجة`) ومحارف لوحات المفاتيح الفارسية/الأردية (`ک/ی ← ك/ي`)، ويرمّم أخطاء التاء المربوطة (`البرمجه ← البرمجة`) دون المساس بالضمائر (`بيته، ملفه، مديره`).
|
|
271
|
+
* **توفير التوكنات وتكاليف الانتباه التربيعي $O(L^2)$:** أظهرت التجارب الموثقة (`misar eval --all-models`) تفوق **100% مقابل 0%** في تمييز الثنائيات الصرفية (`عالمة/عالمه`)، وحفظ **100%** من النفي اللهجي وقطبية المشاعر، وخفض التوكنات بنسبة **46.91%** مقابل الخام و**16.44%** مقابل المناهج التقليدية (ما يوفّر **67.8%–74.3%** من عمليات الانتباه $O(L^2)$ واستعادة `157 → 0 [UNK]`).
|
|
255
272
|
|
|
256
273
|
---
|
|
257
274
|
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "misar"
|
|
7
|
-
version = "2.
|
|
7
|
+
version = "2.1.0"
|
|
8
8
|
description = "Misar (مِسار): Adaptive context-aware Arabic NLP preprocessing router for optimizing Transformer language models."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
license = {text = "MIT"}
|
|
@@ -3,9 +3,13 @@
|
|
|
3
3
|
Misar (مِسار): Adaptive Context-Aware Arabic NLP Preprocessing Router.
|
|
4
4
|
"""
|
|
5
5
|
|
|
6
|
+
__version__ = "2.1.0"
|
|
7
|
+
__author__ = "Muneer Jarab Matrak Al-Theeb"
|
|
8
|
+
|
|
9
|
+
from .evaluation import evaluate_system, run_evaluation
|
|
6
10
|
from .pipelines import DialectPreservePipeline, MSALightPipeline, NoisySocialPipeline
|
|
7
11
|
from .profiler import LinguisticProfile, LinguisticProfiler, compute_shannon_entropy
|
|
8
|
-
from .router import MisarRouter, RoutingResult, preprocess, preprocess_batch, route
|
|
12
|
+
from .router import DocumentRoutingResult, MisarRouter, RoutingResult, preprocess, preprocess_batch, route
|
|
9
13
|
from .tokenizer import (
|
|
10
14
|
PRETRAINED_MODEL_CONFIGS,
|
|
11
15
|
PRETRAINED_VOCAB_URLS,
|
|
@@ -15,14 +19,12 @@ from .tokenizer import (
|
|
|
15
19
|
load_default_tokenizer,
|
|
16
20
|
)
|
|
17
21
|
|
|
18
|
-
__version__ = "2.0.0"
|
|
19
|
-
__author__ = "Muneer Jarab Matrak Al-Theeb"
|
|
20
|
-
|
|
21
22
|
__all__ = [
|
|
22
23
|
"__version__",
|
|
23
24
|
"__author__",
|
|
24
25
|
"MisarRouter",
|
|
25
26
|
"RoutingResult",
|
|
27
|
+
"DocumentRoutingResult",
|
|
26
28
|
"LinguisticProfiler",
|
|
27
29
|
"LinguisticProfile",
|
|
28
30
|
"compute_shannon_entropy",
|
|
@@ -38,4 +40,6 @@ __all__ = [
|
|
|
38
40
|
"route",
|
|
39
41
|
"preprocess",
|
|
40
42
|
"preprocess_batch",
|
|
43
|
+
"evaluate_system",
|
|
44
|
+
"run_evaluation",
|
|
41
45
|
]
|
|
@@ -16,6 +16,7 @@ from typing import List, Optional
|
|
|
16
16
|
|
|
17
17
|
from . import __version__
|
|
18
18
|
from .benchmark import run_benchmark
|
|
19
|
+
from .evaluation import run_evaluation
|
|
19
20
|
from .pipelines import SUPPORTED_EMOJI_MODES
|
|
20
21
|
from .router import MisarRouter
|
|
21
22
|
from .tokenizer import PRETRAINED_MODEL_CONFIGS
|
|
@@ -96,6 +97,12 @@ def cmd_bench(args: argparse.Namespace) -> int:
|
|
|
96
97
|
return 0
|
|
97
98
|
|
|
98
99
|
|
|
100
|
+
def cmd_eval(args: argparse.Namespace) -> int:
|
|
101
|
+
keys = sorted(PRETRAINED_MODEL_CONFIGS.keys()) if args.all_models else [args.model]
|
|
102
|
+
run_evaluation(model_keys=keys, offline=args.offline, json_output=args.json)
|
|
103
|
+
return 0
|
|
104
|
+
|
|
105
|
+
|
|
99
106
|
def build_parser() -> argparse.ArgumentParser:
|
|
100
107
|
parser = argparse.ArgumentParser(
|
|
101
108
|
prog="misar",
|
|
@@ -137,6 +144,13 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
137
144
|
p_bench.add_argument("--save-report", default="", help="Optional JSON output path")
|
|
138
145
|
p_bench.set_defaults(func=cmd_bench)
|
|
139
146
|
|
|
147
|
+
p_eval = sub.add_parser("eval", help="Run semantic disambiguation, negation & cross-model evaluation")
|
|
148
|
+
p_eval.add_argument("--model", default="arabertv02", choices=model_choices)
|
|
149
|
+
p_eval.add_argument("--all-models", action="store_true", help="Evaluate across all 5 Arabic Transformer vocabs")
|
|
150
|
+
p_eval.add_argument("--json", action="store_true", help="Output structured JSON")
|
|
151
|
+
p_eval.add_argument("--offline", action="store_true")
|
|
152
|
+
p_eval.set_defaults(func=cmd_eval)
|
|
153
|
+
|
|
140
154
|
return parser
|
|
141
155
|
|
|
142
156
|
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
"""
|
|
3
|
+
Scientific Downstream & Cross-Model Evaluation Suite for Misar (مِسار).
|
|
4
|
+
Quantifies minimal-pair semantic disambiguation, colloquial negation protection,
|
|
5
|
+
sentiment emoji retention, and quadratic self-attention FLOPs savings across models.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from typing import Any, Dict, List, Optional, Sequence, Tuple
|
|
9
|
+
|
|
10
|
+
from .benchmark import BENCHMARK_CORPUS
|
|
11
|
+
from .router import MisarRouter
|
|
12
|
+
from .tokenizer import PRETRAINED_MODEL_CONFIGS, load_default_tokenizer
|
|
13
|
+
|
|
14
|
+
# Minimal orthographic pairs where legacy static normalization (ة->ه, أ/إ->ا) causes semantic collision.
|
|
15
|
+
SEMANTIC_MINIMAL_PAIRS: Tuple[Tuple[str, str], ...] = (
|
|
16
|
+
("عالمة", "عالمه"),
|
|
17
|
+
("مديرة", "مديره"),
|
|
18
|
+
("سنة", "سنه"),
|
|
19
|
+
("جنة", "جنه"),
|
|
20
|
+
("قصة", "قصه"),
|
|
21
|
+
("كتابة", "كتابه"),
|
|
22
|
+
("حملة", "حمله"),
|
|
23
|
+
("مسلمة", "مسلمه"),
|
|
24
|
+
("مهمة", "مهمه"),
|
|
25
|
+
("أسر", "اسر"),
|
|
26
|
+
)
|
|
27
|
+
|
|
28
|
+
NEGATION_SAFETY_CASES: Tuple[Tuple[str, str], ...] = (
|
|
29
|
+
("المكتبة مو معقدة أبداً", "مو"),
|
|
30
|
+
("السيرفر مب بطيء في الإصدار الجديد", "مب"),
|
|
31
|
+
("النظام ماكو أسرع منه في التشغيل", "ماكو"),
|
|
32
|
+
("الخدمة مافيه أي تأخير الحين", "مافيه"),
|
|
33
|
+
("النسخة الجديدة مافيها أخطاء", "مافيها"),
|
|
34
|
+
("التطبيق مش بطيء في المعالجة", "مش"),
|
|
35
|
+
)
|
|
36
|
+
|
|
37
|
+
SENTIMENT_EMOJI_CASES: Tuple[Tuple[str, str], ...] = (
|
|
38
|
+
("السيرفررررر بطييييء مررررة 😡😡👎", "سلبي"),
|
|
39
|
+
("الإصدارررر الجديييييد ممتاز 😍🔥🚀", "إيجابي"),
|
|
40
|
+
("البرمجه صارت سلسة في المنصه ✅👍", "إيجابي"),
|
|
41
|
+
("في هذه الحياة الواسعة تحديات صعبة 💔😤", "سلبي"),
|
|
42
|
+
)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def evaluate_system(router: Optional[MisarRouter] = None) -> Dict[str, Any]:
|
|
46
|
+
"""Evaluate semantic safety, disambiguation, polarity, and token/attention efficiency."""
|
|
47
|
+
active = router or MisarRouter()
|
|
48
|
+
sentiment_router = MisarRouter(
|
|
49
|
+
tokenizer=active.tokenizer,
|
|
50
|
+
model_key=active.model_key,
|
|
51
|
+
emoji_mode="sentiment",
|
|
52
|
+
auto_heal=active.auto_heal,
|
|
53
|
+
)
|
|
54
|
+
|
|
55
|
+
misar_distinct = 0
|
|
56
|
+
legacy_distinct = 0
|
|
57
|
+
for word_a, word_b in SEMANTIC_MINIMAL_PAIRS:
|
|
58
|
+
if active.preprocess(word_a) != active.preprocess(word_b):
|
|
59
|
+
misar_distinct += 1
|
|
60
|
+
if active._legacy_baseline.process(word_a) != active._legacy_baseline.process(word_b):
|
|
61
|
+
legacy_distinct += 1
|
|
62
|
+
|
|
63
|
+
neg_kept = sum(
|
|
64
|
+
1 for text, marker in NEGATION_SAFETY_CASES if marker in active.preprocess(text)
|
|
65
|
+
)
|
|
66
|
+
sent_kept = sum(
|
|
67
|
+
1
|
|
68
|
+
for text, tag in SENTIMENT_EMOJI_CASES
|
|
69
|
+
if tag in sentiment_router.preprocess(text)
|
|
70
|
+
)
|
|
71
|
+
|
|
72
|
+
tot_raw = 0
|
|
73
|
+
tot_legacy = 0
|
|
74
|
+
tot_misar = 0
|
|
75
|
+
tot_unk_raw = 0
|
|
76
|
+
tot_unk_misar = 0
|
|
77
|
+
for item in BENCHMARK_CORPUS:
|
|
78
|
+
comp = active.compare_baselines(item["text"])
|
|
79
|
+
tot_raw += comp["misar"]["raw_token_count"]
|
|
80
|
+
tot_legacy += comp["legacy_token_count"]
|
|
81
|
+
tot_misar += comp["misar"]["processed_token_count"]
|
|
82
|
+
tot_unk_raw += comp["misar"]["unk_count_raw"]
|
|
83
|
+
tot_unk_misar += comp["misar"]["unk_count_processed"]
|
|
84
|
+
|
|
85
|
+
token_red_raw = ((tot_raw - tot_misar) / tot_raw * 100.0) if tot_raw else 0.0
|
|
86
|
+
token_red_leg = ((tot_legacy - tot_misar) / tot_legacy * 100.0) if tot_legacy else 0.0
|
|
87
|
+
attn_saved_raw = ((1.0 - (tot_misar / tot_raw) ** 2) * 100.0) if tot_raw else 0.0
|
|
88
|
+
|
|
89
|
+
return {
|
|
90
|
+
"model_key": active.model_key,
|
|
91
|
+
"tokenizer_backend": active.tokenizer.backend_name,
|
|
92
|
+
"is_real_tokenizer": active.tokenizer.is_real_vocab,
|
|
93
|
+
"minimal_pairs_tested": len(SEMANTIC_MINIMAL_PAIRS),
|
|
94
|
+
"misar_disambiguation_pct": round(misar_distinct / len(SEMANTIC_MINIMAL_PAIRS) * 100.0, 2),
|
|
95
|
+
"legacy_disambiguation_pct": round(legacy_distinct / len(SEMANTIC_MINIMAL_PAIRS) * 100.0, 2),
|
|
96
|
+
"negation_preservation_pct": round(neg_kept / len(NEGATION_SAFETY_CASES) * 100.0, 2),
|
|
97
|
+
"sentiment_polarity_pct": round(sent_kept / len(SENTIMENT_EMOJI_CASES) * 100.0, 2),
|
|
98
|
+
"total_raw_tokens": tot_raw,
|
|
99
|
+
"total_legacy_tokens": tot_legacy,
|
|
100
|
+
"total_misar_tokens": tot_misar,
|
|
101
|
+
"total_unk_raw": tot_unk_raw,
|
|
102
|
+
"total_unk_misar": tot_unk_misar,
|
|
103
|
+
"token_reduction_vs_raw_pct": round(token_red_raw, 2),
|
|
104
|
+
"token_reduction_vs_legacy_pct": round(token_red_leg, 2),
|
|
105
|
+
"attention_flops_savings_pct": round(attn_saved_raw, 2),
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def run_evaluation(
|
|
110
|
+
model_keys: Optional[Sequence[str]] = None,
|
|
111
|
+
offline: bool = False,
|
|
112
|
+
json_output: bool = False,
|
|
113
|
+
) -> List[Dict[str, Any]]:
|
|
114
|
+
"""Run cross-model scientific evaluation and print a clean summary table."""
|
|
115
|
+
keys = list(model_keys) if model_keys else ["arabertv02"]
|
|
116
|
+
reports: List[Dict[str, Any]] = []
|
|
117
|
+
|
|
118
|
+
for key in keys:
|
|
119
|
+
tok = load_default_tokenizer(model_key=key, allow_download=not offline)
|
|
120
|
+
router = MisarRouter(tokenizer=tok, model_key=key)
|
|
121
|
+
reports.append(evaluate_system(router))
|
|
122
|
+
|
|
123
|
+
if json_output:
|
|
124
|
+
import json
|
|
125
|
+
print(json.dumps(reports, ensure_ascii=False, indent=2))
|
|
126
|
+
return reports
|
|
127
|
+
|
|
128
|
+
print("=" * 110)
|
|
129
|
+
print("Misar Scientific Cross-Model Evaluation | Disambiguation · Polarity · Tokens · O(L^2) Attention Savings")
|
|
130
|
+
print("=" * 110)
|
|
131
|
+
print(f"{'Model Key':<14} | {'Backend':<36} | {'Disamb M/L':<11} | {'Neg%':>5} | {'Sent%':>5} | {'Tok -Raw/-Leg':<16} | {'O(L^2)':>7}")
|
|
132
|
+
print("-" * 110)
|
|
133
|
+
for rep in reports:
|
|
134
|
+
dis = f"{rep['misar_disambiguation_pct']:.0f}% / {rep['legacy_disambiguation_pct']:.0f}%"
|
|
135
|
+
tok_str = f"-{rep['token_reduction_vs_raw_pct']:.1f}% / -{rep['token_reduction_vs_legacy_pct']:.1f}%"
|
|
136
|
+
attn_str = f"-{rep['attention_flops_savings_pct']:.1f}%"
|
|
137
|
+
backend = str(rep["tokenizer_backend"])[:36]
|
|
138
|
+
print(
|
|
139
|
+
f"{rep['model_key']:<14} | {backend:<36} | {dis:<11} | "
|
|
140
|
+
f"{rep['negation_preservation_pct']:>4.0f}% | {rep['sentiment_polarity_pct']:>4.0f}% | "
|
|
141
|
+
f"{tok_str:<16} | {attn_str:>7}"
|
|
142
|
+
)
|
|
143
|
+
print("=" * 110)
|
|
144
|
+
return reports
|