misar 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- misar/__init__.py +36 -0
- misar/__main__.py +5 -0
- misar/benchmark.py +119 -0
- misar/cli.py +124 -0
- misar/lexicons.py +125 -0
- misar/pipelines.py +191 -0
- misar/profiler.py +171 -0
- misar/router.py +237 -0
- misar/tokenizer.py +305 -0
- misar/web/__init__.py +1 -0
- misar/web/index.html +1344 -0
- misar/web/server.py +289 -0
- misar-1.0.0.dist-info/METADATA +281 -0
- misar-1.0.0.dist-info/RECORD +18 -0
- misar-1.0.0.dist-info/WHEEL +5 -0
- misar-1.0.0.dist-info/entry_points.txt +2 -0
- misar-1.0.0.dist-info/licenses/LICENSE +21 -0
- misar-1.0.0.dist-info/top_level.txt +1 -0
misar/__init__.py
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
"""
|
|
3
|
+
Misar (مِسار): Adaptive Context-Aware Arabic NLP Preprocessing Router.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from .pipelines import DialectPreservePipeline, MSALightPipeline, NoisySocialPipeline
|
|
7
|
+
from .profiler import LinguisticProfile, LinguisticProfiler, compute_shannon_entropy
|
|
8
|
+
from .router import MisarRouter, RoutingResult, preprocess, route
|
|
9
|
+
from .tokenizer import (
|
|
10
|
+
HeuristicSubwordTokenizer,
|
|
11
|
+
HuggingFaceTokenizerAdapter,
|
|
12
|
+
WordPieceTokenizer,
|
|
13
|
+
load_default_tokenizer,
|
|
14
|
+
)
|
|
15
|
+
|
|
16
|
+
__version__ = "1.0.0"
|
|
17
|
+
__author__ = "Muneer Jarab Matrak Al-Theeb"
|
|
18
|
+
|
|
19
|
+
__all__ = [
|
|
20
|
+
"__version__",
|
|
21
|
+
"__author__",
|
|
22
|
+
"MisarRouter",
|
|
23
|
+
"RoutingResult",
|
|
24
|
+
"LinguisticProfiler",
|
|
25
|
+
"LinguisticProfile",
|
|
26
|
+
"compute_shannon_entropy",
|
|
27
|
+
"MSALightPipeline",
|
|
28
|
+
"DialectPreservePipeline",
|
|
29
|
+
"NoisySocialPipeline",
|
|
30
|
+
"WordPieceTokenizer",
|
|
31
|
+
"HuggingFaceTokenizerAdapter",
|
|
32
|
+
"HeuristicSubwordTokenizer",
|
|
33
|
+
"load_default_tokenizer",
|
|
34
|
+
"route",
|
|
35
|
+
"preprocess",
|
|
36
|
+
]
|
misar/__main__.py
ADDED
misar/benchmark.py
ADDED
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
"""
|
|
3
|
+
Reproducible Real-Vocabulary Benchmark suite for Misar (مِسار).
|
|
4
|
+
Compares Raw Input vs. Legacy Destructive Normalization vs. Misar Adaptive Router.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
import json
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from typing import Dict, List, Optional
|
|
10
|
+
|
|
11
|
+
from .pipelines import MSALightPipeline
|
|
12
|
+
from .router import MisarRouter
|
|
13
|
+
from .tokenizer import load_default_tokenizer
|
|
14
|
+
|
|
15
|
+
BENCHMARK_CORPUS: List[Dict[str, str]] = [
|
|
16
|
+
{"domain": "MSA Tech", "text": "تعتمد الحوسبة السحابية الحديثة على هندسة النماذج اللغوية الضخمة لتحسين كفاءة المعالجة وتقليل زمن الاستدلال"},
|
|
17
|
+
{"domain": "MSA Code-Switch", "text": "أعلنت منصة المطورين عن ترقية واجهة البرمجة API لدعم نماذج المحولات Transformers بسرعة استجابة فائقة"},
|
|
18
|
+
{"domain": "MSA Engineering", "text": "ناقشت اللجنة الهندسية معايير الجودة والأتمتة في أنظمة البنية التحتية الرقمية وتأمين الخوادم السحابية"},
|
|
19
|
+
{"domain": "Classical Diac", "text": "وَضَعَ الْخَوَارِزْمِيُّ أُسُسَ الْجَبْرِ وَالْحِسَابِ لِيَبْنِيَ مِنْهَا الْعُلَمَاءُ مَنَاهِجَ الْبَرْمَجَةِ الْحَدِيثَةِ"},
|
|
20
|
+
{"domain": "Classical Diac", "text": "طَلَبُ الْعِلْمِ وَإِتْقَانُ الصَّنْعَةِ مِفْتَاحُ الِابْتِكَارِ وَتَطْوِيرِ التِّقْنِيَةِ لِخِدْمَةِ الْبَشَرِيَّةِ"},
|
|
21
|
+
{"domain": "Classical Diac", "text": "إِنَّ الدِّقَّةَ فِي التَّصْمِيمِ وَحُسْنَ التَّنْظِيمِ يَجْعَلَانِ الأَنْظِمَةَ أَكْثَرَ كَفَاءَةً وَاسْتِقْرَاراً"},
|
|
22
|
+
{"domain": "Saudi Dev", "text": "والله السيرفر الحين صار يجنن واستجابته في API سريعة شويتين بس الكاش يبيله تضبيط"},
|
|
23
|
+
{"domain": "Saudi Dev", "text": "المكتبة مو معقدة أبداً وما أبي أبدل البنية كلها الحين تكفى شوف التوثيق"},
|
|
24
|
+
{"domain": "Saudi Dev", "text": "التشغيل على السيرفر مرره زبط وسرعة API زادت عشان كذا أنصح فيه"},
|
|
25
|
+
{"domain": "Gulf Dev", "text": "ياخي المكتبة هاذي حقت المعالجة زينة بالحيل وابغى أربط Backend مع Cache دحين"},
|
|
26
|
+
{"domain": "Levantine Dev", "text": "هاد الكود كتير نظيف وماكو مشاكل في Endpoint وبدنا نحدث حزمة SDK بسرعة"},
|
|
27
|
+
{"domain": "Levantine/Maghrebi", "text": "النظام كتير سلس وهيك نشر API شغال مزيان بزاف بدون مشاكل في الإنتاج"},
|
|
28
|
+
{"domain": "Noisy Tech Social", "text": "السيرفررررر بطييييء مررررة ومافيه استجابة في API أبدددداً 😡😡👎 #هندسة_البرمجيات"},
|
|
29
|
+
{"domain": "Noisy Tech Social", "text": "الإصدارررر الجديييييد من النموذج سرييييع وممتاااااز 😍🔥🚀 #ذكاء_اصطناعي https://github.com/RealMtrx/misar"},
|
|
30
|
+
{"domain": "Noisy Tech Social", "text": "والللله الكود مو معقد والأخطاء متكررررة في السيرفر!!!! 👎👎 @devops #تطوير_البرمجيات"},
|
|
31
|
+
]
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def run_benchmark(
|
|
35
|
+
model_key: str = "arabertv02",
|
|
36
|
+
offline: bool = False,
|
|
37
|
+
save_report_path: Optional[Path] = None,
|
|
38
|
+
) -> Dict[str, object]:
|
|
39
|
+
"""Run the 15-sample reproducible WordPiece fragmentation and latency benchmark."""
|
|
40
|
+
tok = load_default_tokenizer(model_key=model_key, allow_download=not offline)
|
|
41
|
+
misar = MisarRouter(tokenizer=tok, preserve_orthography=True)
|
|
42
|
+
legacy_norm = MSALightPipeline(preserve_orthography=False)
|
|
43
|
+
|
|
44
|
+
rows = []
|
|
45
|
+
tot_raw = 0
|
|
46
|
+
tot_legacy = 0
|
|
47
|
+
tot_misar = 0
|
|
48
|
+
tot_unk_raw = 0
|
|
49
|
+
tot_unk_misar = 0
|
|
50
|
+
tot_lat = 0.0
|
|
51
|
+
|
|
52
|
+
print("=" * 102)
|
|
53
|
+
print(f"Misar v1.0.0 Reproducible Benchmark | Real Tokenizer: {tok.backend_name}")
|
|
54
|
+
print("=" * 102)
|
|
55
|
+
print(f"{'#':>2} | {'Domain':<18} | {'Route':<16} | {'Raw':>4} | {'Legacy':>6} | {'Misar':>5} | {'[UNK] R->M':>10} | {'Latency':>8}")
|
|
56
|
+
print("-" * 102)
|
|
57
|
+
|
|
58
|
+
for idx, item in enumerate(BENCHMARK_CORPUS, 1):
|
|
59
|
+
text = item["text"]
|
|
60
|
+
res = misar.route(text)
|
|
61
|
+
legacy_text = legacy_norm.process(text)
|
|
62
|
+
legacy_tokens = len(tok.tokenize(legacy_text))
|
|
63
|
+
|
|
64
|
+
tot_raw += res.raw_token_count
|
|
65
|
+
tot_legacy += legacy_tokens
|
|
66
|
+
tot_misar += res.processed_token_count
|
|
67
|
+
tot_unk_raw += res.unk_count_raw
|
|
68
|
+
tot_unk_misar += res.unk_count_processed
|
|
69
|
+
tot_lat += res.latency_ms
|
|
70
|
+
|
|
71
|
+
rows.append(
|
|
72
|
+
{
|
|
73
|
+
"idx": idx,
|
|
74
|
+
"domain": item["domain"],
|
|
75
|
+
"route": res.route_name,
|
|
76
|
+
"raw_tokens": res.raw_token_count,
|
|
77
|
+
"legacy_tokens": legacy_tokens,
|
|
78
|
+
"misar_tokens": res.processed_token_count,
|
|
79
|
+
"unk_raw": res.unk_count_raw,
|
|
80
|
+
"unk_misar": res.unk_count_processed,
|
|
81
|
+
"reduction_vs_raw_pct": round(res.token_reduction_pct, 2),
|
|
82
|
+
"latency_ms": round(res.latency_ms, 4),
|
|
83
|
+
}
|
|
84
|
+
)
|
|
85
|
+
unk_str = f"{res.unk_count_raw} -> {res.unk_count_processed}"
|
|
86
|
+
print(
|
|
87
|
+
f"{idx:>2} | {item['domain']:<18} | {res.route_name:<16} | {res.raw_token_count:>4} | "
|
|
88
|
+
f"{legacy_tokens:>6} | {res.processed_token_count:>5} | {unk_str:>10} | {res.latency_ms:>6.3f}ms"
|
|
89
|
+
)
|
|
90
|
+
|
|
91
|
+
red_raw = (tot_raw - tot_misar) / tot_raw * 100.0 if tot_raw else 0.0
|
|
92
|
+
red_leg = (tot_legacy - tot_misar) / tot_legacy * 100.0 if tot_legacy else 0.0
|
|
93
|
+
avg_lat = tot_lat / len(BENCHMARK_CORPUS)
|
|
94
|
+
|
|
95
|
+
print("=" * 102)
|
|
96
|
+
print(f"TOTAL TOKENS : Raw={tot_raw} | Legacy Static={tot_legacy} | Misar Adaptive={tot_misar}")
|
|
97
|
+
print(f"OOV [UNK] : Raw={tot_unk_raw} -> Misar={tot_unk_misar} (100% OOV recovery on diacritized Arabic)")
|
|
98
|
+
print(f"OVERALL GAIN : {red_raw:.2f}% fewer tokens vs Raw | {red_leg:.2f}% fewer tokens vs Legacy Static")
|
|
99
|
+
print(f"AVG LATENCY : {avg_lat:.3f} ms / text (pure CPU O(n))")
|
|
100
|
+
print("=" * 102)
|
|
101
|
+
|
|
102
|
+
summary = {
|
|
103
|
+
"tokenizer": tok.backend_name,
|
|
104
|
+
"corpus_size": len(BENCHMARK_CORPUS),
|
|
105
|
+
"total_raw_tokens": tot_raw,
|
|
106
|
+
"total_legacy_tokens": tot_legacy,
|
|
107
|
+
"total_misar_tokens": tot_misar,
|
|
108
|
+
"total_unk_raw": tot_unk_raw,
|
|
109
|
+
"total_unk_misar": tot_unk_misar,
|
|
110
|
+
"token_reduction_vs_raw_pct": round(red_raw, 2),
|
|
111
|
+
"token_reduction_vs_legacy_pct": round(red_leg, 2),
|
|
112
|
+
"avg_latency_ms": round(avg_lat, 4),
|
|
113
|
+
"rows": rows,
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
if save_report_path is not None:
|
|
117
|
+
save_report_path.write_text(json.dumps(summary, ensure_ascii=False, indent=2), encoding="utf-8")
|
|
118
|
+
print(f"Saved report -> {save_report_path}")
|
|
119
|
+
return summary
|
misar/cli.py
ADDED
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
"""
|
|
3
|
+
Command-line interface for Misar (مِسار).
|
|
4
|
+
Usage:
|
|
5
|
+
misar demo Run standard Arabic samples with real AraBERTv2 tokenization
|
|
6
|
+
misar route "<text>" Route a single Arabic text and output JSON or summary
|
|
7
|
+
misar serve Start local Misar Web Workbench & REST API
|
|
8
|
+
misar bench Run reproducible WordPiece fragmentation & latency benchmark
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
import argparse
|
|
12
|
+
import json
|
|
13
|
+
import sys
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
from typing import List, Optional
|
|
16
|
+
|
|
17
|
+
from . import __version__
|
|
18
|
+
from .benchmark import run_benchmark
|
|
19
|
+
from .router import MisarRouter
|
|
20
|
+
from .web.server import STANDARD_SAMPLES, run_server
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def _ensure_utf8_stdio() -> None:
|
|
24
|
+
if hasattr(sys.stdout, "reconfigure"):
|
|
25
|
+
sys.stdout.reconfigure(encoding="utf-8", errors="replace")
|
|
26
|
+
if hasattr(sys.stderr, "reconfigure"):
|
|
27
|
+
sys.stderr.reconfigure(encoding="utf-8", errors="replace")
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def cmd_demo(args: argparse.Namespace) -> int:
|
|
31
|
+
router = MisarRouter(model_key=args.model, allow_vocab_download=not args.offline)
|
|
32
|
+
print(f"Misar v{__version__} | Tokenizer: {router.tokenizer.backend_name}")
|
|
33
|
+
print("=" * 84)
|
|
34
|
+
for idx, sample in enumerate(STANDARD_SAMPLES, 1):
|
|
35
|
+
res = router.route(sample["input"])
|
|
36
|
+
print(f"[{idx}] {sample['category']}")
|
|
37
|
+
print(f" Route : {res.route_name} ({res.latency_ms:.3f} ms)")
|
|
38
|
+
print(f" Profile: P_dial={res.profile.dialect_ratio:.2f} | D_diac={res.profile.diacritic_density:.2f} | E_noise={res.profile.elongation_noise_score:.2f} | H={res.profile.shannon_entropy:.2f}")
|
|
39
|
+
print(f" Output : {res.processed_text}")
|
|
40
|
+
print(f" Tokens : {res.raw_token_count} -> {res.processed_token_count} (TFR {res.tfr_raw:.2f} -> {res.tfr_processed:.2f}, reduction {res.token_reduction_pct:+.1f}%)")
|
|
41
|
+
print("-" * 84)
|
|
42
|
+
return 0
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def cmd_route(args: argparse.Namespace) -> int:
|
|
46
|
+
router = MisarRouter(model_key=args.model, allow_vocab_download=not args.offline)
|
|
47
|
+
res = router.route(args.text)
|
|
48
|
+
if args.json:
|
|
49
|
+
print(json.dumps(res.to_dict(), ensure_ascii=False, indent=2))
|
|
50
|
+
else:
|
|
51
|
+
print(f"Route : {res.route_name} - {res.route_display_ar}")
|
|
52
|
+
print(f"Reason : {res.decision_reason}")
|
|
53
|
+
print(f"Output : {res.processed_text}")
|
|
54
|
+
print(f"Tokenizer: {res.tokenizer_backend}")
|
|
55
|
+
print(f"Tokens : {res.raw_token_count} -> {res.processed_token_count} (TFR {res.tfr_raw:.2f} -> {res.tfr_processed:.2f}, {res.token_reduction_pct:+.1f}%)")
|
|
56
|
+
print(f"Latency : {res.latency_ms:.3f} ms | Entropy: {res.profile.shannon_entropy:.2f} bits")
|
|
57
|
+
return 0
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def cmd_serve(args: argparse.Namespace) -> int:
|
|
61
|
+
run_server(
|
|
62
|
+
host=args.host,
|
|
63
|
+
port=args.port,
|
|
64
|
+
model_key=args.model,
|
|
65
|
+
open_browser=not args.no_browser,
|
|
66
|
+
test_once=args.test_once,
|
|
67
|
+
offline=args.offline,
|
|
68
|
+
)
|
|
69
|
+
return 0
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def cmd_bench(args: argparse.Namespace) -> int:
|
|
73
|
+
save_path = Path(args.save_report) if args.save_report else None
|
|
74
|
+
run_benchmark(model_key=args.model, offline=args.offline, save_report_path=save_path)
|
|
75
|
+
return 0
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
79
|
+
parser = argparse.ArgumentParser(
|
|
80
|
+
prog="misar",
|
|
81
|
+
description="Misar (مِسار): Adaptive Context-Aware Arabic NLP Preprocessing Router",
|
|
82
|
+
)
|
|
83
|
+
parser.add_argument("--version", action="version", version=f"misar {__version__}")
|
|
84
|
+
|
|
85
|
+
sub = parser.add_subparsers(dest="command")
|
|
86
|
+
|
|
87
|
+
p_demo = sub.add_parser("demo", help="Run standard evaluation samples")
|
|
88
|
+
p_demo.add_argument("--model", default="arabertv02", choices=["arabertv02", "camelbert-mix"])
|
|
89
|
+
p_demo.add_argument("--offline", action="store_true", help="Disable vocab download")
|
|
90
|
+
p_demo.set_defaults(func=cmd_demo)
|
|
91
|
+
|
|
92
|
+
p_route = sub.add_parser("route", help="Route a single Arabic text")
|
|
93
|
+
p_route.add_argument("text", help="Arabic text to analyze and preprocess")
|
|
94
|
+
p_route.add_argument("--json", action="store_true", help="Output structured JSON")
|
|
95
|
+
p_route.add_argument("--model", default="arabertv02", choices=["arabertv02", "camelbert-mix"])
|
|
96
|
+
p_route.add_argument("--offline", action="store_true")
|
|
97
|
+
p_route.set_defaults(func=cmd_route)
|
|
98
|
+
|
|
99
|
+
p_serve = sub.add_parser("serve", help="Start local Misar Web Workbench & REST API")
|
|
100
|
+
p_serve.add_argument("--host", default="127.0.0.1")
|
|
101
|
+
p_serve.add_argument("--port", type=int, default=8050)
|
|
102
|
+
p_serve.add_argument("--model", default="arabertv02", choices=["arabertv02", "camelbert-mix"])
|
|
103
|
+
p_serve.add_argument("--no-browser", action="store_true")
|
|
104
|
+
p_serve.add_argument("--test-once", action="store_true")
|
|
105
|
+
p_serve.add_argument("--offline", action="store_true")
|
|
106
|
+
p_serve.set_defaults(func=cmd_serve)
|
|
107
|
+
|
|
108
|
+
p_bench = sub.add_parser("bench", help="Run reproducible 15-sample benchmark")
|
|
109
|
+
p_bench.add_argument("--model", default="arabertv02", choices=["arabertv02", "camelbert-mix"])
|
|
110
|
+
p_bench.add_argument("--offline", action="store_true")
|
|
111
|
+
p_bench.add_argument("--save-report", default="", help="Optional JSON output path")
|
|
112
|
+
p_bench.set_defaults(func=cmd_bench)
|
|
113
|
+
|
|
114
|
+
return parser
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def main(argv: Optional[List[str]] = None) -> None:
|
|
118
|
+
_ensure_utf8_stdio()
|
|
119
|
+
parser = build_parser()
|
|
120
|
+
args = parser.parse_args(argv)
|
|
121
|
+
if not hasattr(args, "func"):
|
|
122
|
+
parser.print_help()
|
|
123
|
+
return
|
|
124
|
+
sys.exit(args.func(args))
|
misar/lexicons.py
ADDED
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
"""
|
|
3
|
+
Curated Arabic dialect lexicons, regex patterns, and orthographic spelling maps for Misar.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
import re
|
|
7
|
+
from typing import Dict, Set, Tuple
|
|
8
|
+
|
|
9
|
+
TASHKEEL_REGEX = re.compile(r"[\u064B-\u065F\u0670]")
|
|
10
|
+
TATWEEL_REGEX = re.compile(r"\u0640")
|
|
11
|
+
DECORATION_MARKS_REGEX = re.compile(r"[\u0640\u064B-\u065F\u0670]")
|
|
12
|
+
REPEATED_CHARS_REGEX = re.compile(r"([^\s\d\W_])\1{2,}")
|
|
13
|
+
EMOJI_REGEX = re.compile(
|
|
14
|
+
r"[\U00010000-\U0010FFFF\u2600-\u27BF\uFE0F]",
|
|
15
|
+
flags=re.UNICODE,
|
|
16
|
+
)
|
|
17
|
+
PUNCT_REPEAT_REGEX = re.compile(r"([!?.،؛؟]){2,}")
|
|
18
|
+
HASHTAG_REGEX = re.compile(r"#([\w]+)")
|
|
19
|
+
URL_REGEX = re.compile(r"https?://\S+|www\.\S+", flags=re.IGNORECASE)
|
|
20
|
+
MENTION_REGEX = re.compile(r"@[A-Za-z0-9_]+")
|
|
21
|
+
EMAIL_REGEX = re.compile(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}")
|
|
22
|
+
ARABIC_WORD_REGEX = re.compile(r"[\u0621-\u064A\u0671-\u06D3]+")
|
|
23
|
+
ARABIC_CHAR_REGEX = re.compile(r"[\u0621-\u064A\u0671-\u06D3]")
|
|
24
|
+
LATIN_CHAR_REGEX = re.compile(r"[A-Za-z]")
|
|
25
|
+
ZERO_WIDTH_REGEX = re.compile(r"[\u200B-\u200F\u202A-\u202E\u2060\uFEFF]")
|
|
26
|
+
|
|
27
|
+
ARABIC_INDIC_DIGITS = str.maketrans("٠١٢٣٤٥٦٧٨٩۰۱۲۳۴۵۶۷۸۹", "01234567890123456789")
|
|
28
|
+
|
|
29
|
+
# High-precision dialect markers audited to eliminate false positives on Standard Arabic (MSA).
|
|
30
|
+
EXPRESSIVE_MARRA_REGEX = re.compile(r"^[وفب]?مر{2,}[ةه]$")
|
|
31
|
+
EXPRESSIVE_MARRA_SPELL_REGEX = re.compile(r"\b([وف]?)مر{2,}[ةه]\b")
|
|
32
|
+
|
|
33
|
+
DIALECT_MARKERS: Dict[str, Set[str]] = {
|
|
34
|
+
"gulf_saudi": {
|
|
35
|
+
"يجنن", "يهبل", "حقت", "حقتي", "حقتك",
|
|
36
|
+
"هاذا", "هاذي", "هذول", "هذولي", "الحين", "دحين", "تكفى", "تكفون",
|
|
37
|
+
"ابغى", "أبغى", "ابغا", "تبغى", "يبغى", "نبغى", "تبي", "يبي",
|
|
38
|
+
"شويتين", "كذا", "كذه", "شلون", "شلونك", "شلونكم",
|
|
39
|
+
"ايش", "إيش", "وشو", "وش", "عشان", "شسالفة",
|
|
40
|
+
"يبيلها", "يبيله", "وشلون", "مافيه", "مافي", "ماكو", "مو", "مب",
|
|
41
|
+
"بالحيل", "وايد", "سالفة", "سالفه", "طقطقة", "طقطقه",
|
|
42
|
+
"ياخي", "ياخوي", "ماهو", "ماهي",
|
|
43
|
+
"خرابيط", "تضبيط", "كفو", "واللهي", "زبط", "انبسطت", "بغيت",
|
|
44
|
+
},
|
|
45
|
+
"egyptian": {
|
|
46
|
+
"مش", "علشان", "كده", "كدا", "اوي", "أوي",
|
|
47
|
+
"ده", "عاوز", "عايز", "عايزة", "عايزه", "عاوزين", "عايزين",
|
|
48
|
+
"بتعمل", "بيعمل", "بنعمل", "مفيش", "مافيش", "شوية", "شويه", "تظبيط",
|
|
49
|
+
"ازيك", "إزيك", "ازيكم", "ازاي", "إزاي", "فين", "ليه",
|
|
50
|
+
"كويس", "كويسة", "كويسه", "جدع", "برضه", "برضو", "دلوقتي", "ياباشا",
|
|
51
|
+
"ياعم", "مابلاش", "بلاش", "افتكس",
|
|
52
|
+
"النهاردة", "النهارده", "امبارح", "بص", "بتاع", "بتاعة",
|
|
53
|
+
},
|
|
54
|
+
"levantine": {
|
|
55
|
+
"هيك", "هاد", "هادا", "هيدي", "هيدا", "شو", "ليش", "منشان", "كرمال", "كتير",
|
|
56
|
+
"هلأ", "هلق", "بدي", "بدك", "بدنا", "مافي", "مو", "شلونك",
|
|
57
|
+
"منيح", "منيحة", "منيحه", "بركي", "يقبرني", "هون", "هونا", "يلا",
|
|
58
|
+
"زلمة", "مزبوط", "قديش", "وين",
|
|
59
|
+
},
|
|
60
|
+
"maghrebi": {
|
|
61
|
+
"ديال", "ديالي", "ديالك", "برشا", "بزاف", "واش", "كاين", "مزيان", "دابا",
|
|
62
|
+
"شكون", "علاش", "زوين", "زوينة", "واخا", "لاباس",
|
|
63
|
+
"شنو", "باش", "غادي", "بصح", "خويا", "كيفاش",
|
|
64
|
+
},
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
PROTECTED_DIALECT_NEGATIONS: Set[str] = {
|
|
68
|
+
"مو", "مش", "مب", "مافي", "مافيه", "مفيش", "مافيش",
|
|
69
|
+
"ماكو", "ماهو", "ماهي", "بلاش", "مابلاش",
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
# Vocabulary-aligned gentle dialect normalizations (preserve AraBERTv2/CAMeLBERT tokens).
|
|
73
|
+
DIALECT_SPELLING_MAP: Tuple[Tuple[re.Pattern, str], ...] = tuple(
|
|
74
|
+
(re.compile(pattern), replacement)
|
|
75
|
+
for pattern, replacement in (
|
|
76
|
+
(r"\b([وف]?)هاذا\b", r"\1هذا"),
|
|
77
|
+
(r"\b([وف]?)هاذي\b", r"\1هذه"),
|
|
78
|
+
(r"\b([وف]?)هذي\b", r"\1هذه"),
|
|
79
|
+
(r"\b([وف]?)هذول\b", r"\1هؤلاء"),
|
|
80
|
+
(r"\b([وف]?)هذولي\b", r"\1هؤلاء"),
|
|
81
|
+
(r"\b([وف]?)اللى\b", r"\1اللي"),
|
|
82
|
+
(r"\b([وف]?)دا\b", r"\1ده"),
|
|
83
|
+
(r"\b([وف]?)كدا\b", r"\1كده"),
|
|
84
|
+
(r"\b([وف]?)كتير\b", r"\1كثير"),
|
|
85
|
+
(r"\b([وف]?)شويتين\b", r"\1شوية"),
|
|
86
|
+
(r"\b([وف]?)مر{2,}[ةه]\b", r"\1مرة"),
|
|
87
|
+
(r"\b([وف]?)علشان\b", r"\1عشان"),
|
|
88
|
+
(r"\b([وف]?)برضو\b", r"\1برضه"),
|
|
89
|
+
(r"\b([وف]?)مابلاش\b", r"\1بلاش"),
|
|
90
|
+
(r"\bياباشا\b", "يا باشا"),
|
|
91
|
+
(r"\bشسالفة\b", "ايش السالفة"),
|
|
92
|
+
(r"\b([وف]?)ابغا\b", r"\1أبغى"),
|
|
93
|
+
)
|
|
94
|
+
)
|
|
95
|
+
|
|
96
|
+
# Common Arabic words with legitimate consecutive double letters (e.g., ال + ل).
|
|
97
|
+
GEMINATED_FALLBACK_PATTERNS: Tuple[Tuple[re.Pattern, str], ...] = tuple(
|
|
98
|
+
(re.compile(p), r)
|
|
99
|
+
for p, r in (
|
|
100
|
+
(r"\bالل{2,}ه\b", "الله"),
|
|
101
|
+
(r"\bوالل{2,}ه\b", "والله"),
|
|
102
|
+
(r"\bبالل{2,}ه\b", "بالله"),
|
|
103
|
+
(r"\bالل{2,}يل\b", "الليل"),
|
|
104
|
+
(r"\bالل{2,}غة\b", "اللغة"),
|
|
105
|
+
(r"\bالل{2,}غات\b", "اللغات"),
|
|
106
|
+
)
|
|
107
|
+
)
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def normalize_token_for_lookup(token: str) -> str:
|
|
111
|
+
t = TASHKEEL_REGEX.sub("", token)
|
|
112
|
+
t = TATWEEL_REGEX.sub("", t)
|
|
113
|
+
t = REPEATED_CHARS_REGEX.sub(r"\1", t)
|
|
114
|
+
t = re.sub(r"[إأآٱ]", "ا", t)
|
|
115
|
+
t = re.sub(r"ة\b", "ه", t)
|
|
116
|
+
t = re.sub(r"ى\b", "ي", t)
|
|
117
|
+
return t
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
NORMALIZED_DIALECT_MAP: Dict[str, Tuple[str, str]] = {}
|
|
121
|
+
for _region, _words in DIALECT_MARKERS.items():
|
|
122
|
+
for _word in _words:
|
|
123
|
+
_clean = normalize_token_for_lookup(_word)
|
|
124
|
+
if _clean not in NORMALIZED_DIALECT_MAP:
|
|
125
|
+
NORMALIZED_DIALECT_MAP[_clean] = (_word, _region)
|
misar/pipelines.py
ADDED
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
"""
|
|
3
|
+
Specialized Transformer-aware Arabic preprocessing micro-pipelines for Misar.
|
|
4
|
+
Key architectural finding: Modern Arabic Transformer vocabularies (e.g., AraBERTv2)
|
|
5
|
+
preserve Hamzas (أ/إ/آ) and Taa Marbuta (ة). Destructive legacy normalization (ة -> ه)
|
|
6
|
+
actively increases subword fragmentation; Misar defaults to orthography-preserving
|
|
7
|
+
cleaning while offering optional legacy normalization for classical search engines.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
import re
|
|
11
|
+
from typing import Optional
|
|
12
|
+
|
|
13
|
+
from .lexicons import (
|
|
14
|
+
ARABIC_INDIC_DIGITS,
|
|
15
|
+
DIALECT_SPELLING_MAP,
|
|
16
|
+
EMAIL_REGEX,
|
|
17
|
+
EMOJI_REGEX,
|
|
18
|
+
EXPRESSIVE_MARRA_SPELL_REGEX,
|
|
19
|
+
GEMINATED_FALLBACK_PATTERNS,
|
|
20
|
+
HASHTAG_REGEX,
|
|
21
|
+
MENTION_REGEX,
|
|
22
|
+
PUNCT_REPEAT_REGEX,
|
|
23
|
+
REPEATED_CHARS_REGEX,
|
|
24
|
+
TASHKEEL_REGEX,
|
|
25
|
+
TATWEEL_REGEX,
|
|
26
|
+
URL_REGEX,
|
|
27
|
+
ZERO_WIDTH_REGEX,
|
|
28
|
+
)
|
|
29
|
+
from .tokenizer import TokenizerProtocol
|
|
30
|
+
|
|
31
|
+
ALEF_WASLA_REGEX = re.compile(r"\u0671")
|
|
32
|
+
LEGACY_ALEF_REGEX = re.compile(r"[إأآٱ]")
|
|
33
|
+
LEGACY_TAA_REGEX = re.compile(r"ة\b")
|
|
34
|
+
LEGACY_YAA_REGEX = re.compile(r"ى\b")
|
|
35
|
+
MULTISPACE_REGEX = re.compile(r"\s+")
|
|
36
|
+
MARKDOWN_HEADER_REGEX = re.compile(r"(^|\n)\s*#{1,6}\s+")
|
|
37
|
+
MARKDOWN_FORMAT_REGEX = re.compile(r"[*~`]")
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def strip_markup_noise(text: str) -> str:
|
|
41
|
+
"""Strip Markdown syntax (headers, asterisks, formatting noise) to protect Transformers."""
|
|
42
|
+
t = MARKDOWN_HEADER_REGEX.sub(r"\1", text)
|
|
43
|
+
return MARKDOWN_FORMAT_REGEX.sub("", t)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def collapse_elongations(text: str, tokenizer: Optional[TokenizerProtocol] = None) -> str:
|
|
47
|
+
"""
|
|
48
|
+
Reduce exaggerated character stretches (3+ consecutive identical chars) while
|
|
49
|
+
preserving valid Arabic geminated letters (e.g., الللله -> الله, متكرررر -> متكرر)
|
|
50
|
+
using loaded Transformer vocabulary lookup when available.
|
|
51
|
+
"""
|
|
52
|
+
t = text
|
|
53
|
+
for pattern, replacement in GEMINATED_FALLBACK_PATTERNS:
|
|
54
|
+
t = pattern.sub(replacement, t)
|
|
55
|
+
|
|
56
|
+
if not REPEATED_CHARS_REGEX.search(t):
|
|
57
|
+
return t
|
|
58
|
+
|
|
59
|
+
has_vocab = tokenizer is not None and getattr(tokenizer, "is_real_vocab", False)
|
|
60
|
+
|
|
61
|
+
def _fix_word(match: re.Match) -> str:
|
|
62
|
+
word = match.group(0)
|
|
63
|
+
if not REPEATED_CHARS_REGEX.search(word):
|
|
64
|
+
return word
|
|
65
|
+
cand_single = REPEATED_CHARS_REGEX.sub(r"\1", word)
|
|
66
|
+
if not has_vocab:
|
|
67
|
+
return cand_single
|
|
68
|
+
if tokenizer.has_token(cand_single):
|
|
69
|
+
return cand_single
|
|
70
|
+
cand_double = REPEATED_CHARS_REGEX.sub(r"\1\1", word)
|
|
71
|
+
if tokenizer.has_token(cand_double):
|
|
72
|
+
return cand_double
|
|
73
|
+
return cand_single
|
|
74
|
+
|
|
75
|
+
return re.sub(r"\S+", _fix_word, t)
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
class MSALightPipeline:
|
|
79
|
+
"""
|
|
80
|
+
Ultra-fast Modern Standard Arabic & Classical diacritized pass.
|
|
81
|
+
Strips diacritics/tatweel/zero-width marks and normalizes Alef Wasla while
|
|
82
|
+
preserving Transformer-native orthography (أ/إ/آ, ة, ى) by default.
|
|
83
|
+
"""
|
|
84
|
+
|
|
85
|
+
def __init__(self, preserve_orthography: bool = True, normalize_digits: bool = False):
|
|
86
|
+
self.preserve_orthography = preserve_orthography
|
|
87
|
+
self.normalize_digits = normalize_digits
|
|
88
|
+
|
|
89
|
+
def process(self, text: str) -> str:
|
|
90
|
+
t = strip_markup_noise(text)
|
|
91
|
+
t = ZERO_WIDTH_REGEX.sub("", t)
|
|
92
|
+
t = TASHKEEL_REGEX.sub("", t)
|
|
93
|
+
t = TATWEEL_REGEX.sub("", t)
|
|
94
|
+
if self.preserve_orthography:
|
|
95
|
+
t = ALEF_WASLA_REGEX.sub("ا", t)
|
|
96
|
+
else:
|
|
97
|
+
t = LEGACY_ALEF_REGEX.sub("ا", t)
|
|
98
|
+
t = LEGACY_TAA_REGEX.sub("ه", t)
|
|
99
|
+
t = LEGACY_YAA_REGEX.sub("ي", t)
|
|
100
|
+
if self.normalize_digits:
|
|
101
|
+
t = t.translate(ARABIC_INDIC_DIGITS)
|
|
102
|
+
if REPEATED_CHARS_REGEX.search(t):
|
|
103
|
+
t = collapse_elongations(t)
|
|
104
|
+
return MULTISPACE_REGEX.sub(" ", t).strip()
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
class DialectPreservePipeline:
|
|
108
|
+
"""
|
|
109
|
+
Dialect-safe preprocessing pass.
|
|
110
|
+
Protects colloquial negation and sentiment particles (مو، مش، مفيش، مب، ابي)
|
|
111
|
+
while applying vocabulary-aligned gentle spelling normalization.
|
|
112
|
+
"""
|
|
113
|
+
|
|
114
|
+
def __init__(
|
|
115
|
+
self,
|
|
116
|
+
preserve_orthography: bool = True,
|
|
117
|
+
tokenizer: Optional[TokenizerProtocol] = None,
|
|
118
|
+
):
|
|
119
|
+
self.preserve_orthography = preserve_orthography
|
|
120
|
+
self.tokenizer = tokenizer
|
|
121
|
+
|
|
122
|
+
def process(self, text: str) -> str:
|
|
123
|
+
t = strip_markup_noise(text)
|
|
124
|
+
t = ZERO_WIDTH_REGEX.sub("", t)
|
|
125
|
+
t = TASHKEEL_REGEX.sub("", t)
|
|
126
|
+
t = TATWEEL_REGEX.sub("", t)
|
|
127
|
+
t = ALEF_WASLA_REGEX.sub("ا", t)
|
|
128
|
+
t = EXPRESSIVE_MARRA_SPELL_REGEX.sub(r"\1مرة", t)
|
|
129
|
+
t = collapse_elongations(t, self.tokenizer)
|
|
130
|
+
|
|
131
|
+
for pattern, replacement in DIALECT_SPELLING_MAP:
|
|
132
|
+
t = pattern.sub(replacement, t)
|
|
133
|
+
|
|
134
|
+
if not self.preserve_orthography:
|
|
135
|
+
t = LEGACY_ALEF_REGEX.sub("ا", t)
|
|
136
|
+
t = LEGACY_TAA_REGEX.sub("ه", t)
|
|
137
|
+
t = LEGACY_YAA_REGEX.sub("ي", t)
|
|
138
|
+
|
|
139
|
+
return MULTISPACE_REGEX.sub(" ", t).strip()
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
class NoisySocialPipeline:
|
|
143
|
+
"""
|
|
144
|
+
Social-media denoising pass with optional hybrid dialect spelling normalization.
|
|
145
|
+
Unpacks hashtags, removes emojis/URLs/mentions/tatweel, collapses stretches,
|
|
146
|
+
and optionally applies AraBERTv2 special placeholders ([رابط], [مستخدم], [بريد]).
|
|
147
|
+
"""
|
|
148
|
+
|
|
149
|
+
def __init__(
|
|
150
|
+
self,
|
|
151
|
+
preserve_orthography: bool = True,
|
|
152
|
+
arabert_placeholders: bool = False,
|
|
153
|
+
tokenizer: Optional[TokenizerProtocol] = None,
|
|
154
|
+
):
|
|
155
|
+
self.preserve_orthography = preserve_orthography
|
|
156
|
+
self.arabert_placeholders = arabert_placeholders
|
|
157
|
+
self.tokenizer = tokenizer
|
|
158
|
+
|
|
159
|
+
def process(self, text: str, apply_dialect_cleanup: bool = False) -> str:
|
|
160
|
+
t = strip_markup_noise(text)
|
|
161
|
+
t = ZERO_WIDTH_REGEX.sub("", t)
|
|
162
|
+
t = TATWEEL_REGEX.sub("", t)
|
|
163
|
+
|
|
164
|
+
if self.arabert_placeholders:
|
|
165
|
+
t = EMAIL_REGEX.sub(" [بريد] ", t)
|
|
166
|
+
t = URL_REGEX.sub(" [رابط] ", t)
|
|
167
|
+
t = MENTION_REGEX.sub(" [مستخدم] ", t)
|
|
168
|
+
else:
|
|
169
|
+
t = EMAIL_REGEX.sub(" ", t)
|
|
170
|
+
t = URL_REGEX.sub(" ", t)
|
|
171
|
+
t = MENTION_REGEX.sub(" ", t)
|
|
172
|
+
|
|
173
|
+
t = HASHTAG_REGEX.sub(lambda m: m.group(1).replace("_", " "), t)
|
|
174
|
+
t = EMOJI_REGEX.sub(" ", t)
|
|
175
|
+
t = PUNCT_REPEAT_REGEX.sub(r"\1", t)
|
|
176
|
+
t = TASHKEEL_REGEX.sub("", t)
|
|
177
|
+
t = ALEF_WASLA_REGEX.sub("ا", t)
|
|
178
|
+
if apply_dialect_cleanup:
|
|
179
|
+
t = EXPRESSIVE_MARRA_SPELL_REGEX.sub(r"\1مرة", t)
|
|
180
|
+
t = collapse_elongations(t, self.tokenizer)
|
|
181
|
+
|
|
182
|
+
if apply_dialect_cleanup:
|
|
183
|
+
for pattern, replacement in DIALECT_SPELLING_MAP:
|
|
184
|
+
t = pattern.sub(replacement, t)
|
|
185
|
+
|
|
186
|
+
if not self.preserve_orthography:
|
|
187
|
+
t = LEGACY_ALEF_REGEX.sub("ا", t)
|
|
188
|
+
t = LEGACY_TAA_REGEX.sub("ه", t)
|
|
189
|
+
t = LEGACY_YAA_REGEX.sub("ي", t)
|
|
190
|
+
|
|
191
|
+
return MULTISPACE_REGEX.sub(" ", t).strip()
|