misar 2.0.0__tar.gz → 2.0.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {misar-2.0.0/src/misar.egg-info → misar-2.0.1}/PKG-INFO +1 -1
- {misar-2.0.0 → misar-2.0.1}/pyproject.toml +1 -1
- {misar-2.0.0 → misar-2.0.1}/src/misar/__init__.py +1 -1
- {misar-2.0.0 → misar-2.0.1}/src/misar/lexicons.py +2 -2
- {misar-2.0.0 → misar-2.0.1}/src/misar/pipelines.py +9 -2
- {misar-2.0.0 → misar-2.0.1}/src/misar/router.py +4 -4
- {misar-2.0.0 → misar-2.0.1}/src/misar/web/index.html +35 -12
- {misar-2.0.0 → misar-2.0.1/src/misar.egg-info}/PKG-INFO +1 -1
- {misar-2.0.0 → misar-2.0.1}/tests/test_misar.py +12 -0
- {misar-2.0.0 → misar-2.0.1}/LICENSE +0 -0
- {misar-2.0.0 → misar-2.0.1}/README.md +0 -0
- {misar-2.0.0 → misar-2.0.1}/setup.cfg +0 -0
- {misar-2.0.0 → misar-2.0.1}/src/misar/__main__.py +0 -0
- {misar-2.0.0 → misar-2.0.1}/src/misar/benchmark.py +0 -0
- {misar-2.0.0 → misar-2.0.1}/src/misar/cli.py +0 -0
- {misar-2.0.0 → misar-2.0.1}/src/misar/profiler.py +0 -0
- {misar-2.0.0 → misar-2.0.1}/src/misar/tokenizer.py +0 -0
- {misar-2.0.0 → misar-2.0.1}/src/misar/web/__init__.py +0 -0
- {misar-2.0.0 → misar-2.0.1}/src/misar/web/server.py +0 -0
- {misar-2.0.0 → misar-2.0.1}/src/misar.egg-info/SOURCES.txt +0 -0
- {misar-2.0.0 → misar-2.0.1}/src/misar.egg-info/dependency_links.txt +0 -0
- {misar-2.0.0 → misar-2.0.1}/src/misar.egg-info/entry_points.txt +0 -0
- {misar-2.0.0 → misar-2.0.1}/src/misar.egg-info/requires.txt +0 -0
- {misar-2.0.0 → misar-2.0.1}/src/misar.egg-info/top_level.txt +0 -0
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "misar"
|
|
7
|
-
version = "2.0.
|
|
7
|
+
version = "2.0.1"
|
|
8
8
|
description = "Misar (مِسار): Adaptive context-aware Arabic NLP preprocessing router for optimizing Transformer language models."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
license = {text = "MIT"}
|
|
@@ -11,7 +11,7 @@ TATWEEL_REGEX = re.compile(r"\u0640")
|
|
|
11
11
|
DECORATION_MARKS_REGEX = re.compile(r"[\u0640\u064B-\u065F\u0670]")
|
|
12
12
|
REPEATED_CHARS_REGEX = re.compile(r"([^\s\d\W_])\1{2,}")
|
|
13
13
|
EMOJI_REGEX = re.compile(
|
|
14
|
-
r"[\U00010000-\U0010FFFF\u2600-\u27BF\uFE0F]",
|
|
14
|
+
r"[\U00010000-\U0010FFFF\u2600-\u27BF\u2300-\u23FF\u2B00-\u2BFF\uFE0F]",
|
|
15
15
|
flags=re.UNICODE,
|
|
16
16
|
)
|
|
17
17
|
PUNCT_REPEAT_REGEX = re.compile(r"([!?.،؛؟]){2,}")
|
|
@@ -39,7 +39,7 @@ POSSESSIVE_AMBIGUOUS_H_WORDS: Set[str] = {
|
|
|
39
39
|
|
|
40
40
|
POSITIVE_EMOJIS: Set[str] = {
|
|
41
41
|
"😍", "🥰", "😘", "😊", "😁", "😄", "😃", "🥳", "🤩", "🙌", "👏",
|
|
42
|
-
"👍", "👌", "🔥", "🚀", "✨", "🌟", "💯", "✅", "♥", "❤", "💖", "💙", "💚",
|
|
42
|
+
"👍", "👌", "🔥", "🚀", "✨", "🌟", "⭐", "💯", "✅", "♥", "❤", "💖", "💙", "💚",
|
|
43
43
|
}
|
|
44
44
|
|
|
45
45
|
NEGATIVE_EMOJIS: Set[str] = {
|
|
@@ -43,7 +43,7 @@ MULTISPACE_REGEX = re.compile(r"\s+")
|
|
|
43
43
|
MARKDOWN_HEADER_REGEX = re.compile(r"(^|\n)\s*#{1,6}\s+")
|
|
44
44
|
MARKDOWN_FORMAT_REGEX = re.compile(r"[*~`]")
|
|
45
45
|
ARABIC_FINAL_HAA_WORD_REGEX = re.compile(r"\b([\u0621-\u064A]{4,})\b")
|
|
46
|
-
EMOJI_RUN_REGEX = re.compile(r"[\U00010000-\U0010FFFF\u2600-\u27BF\uFE0F\u200D]+", flags=re.UNICODE)
|
|
46
|
+
EMOJI_RUN_REGEX = re.compile(r"[\U00010000-\U0010FFFF\u2600-\u27BF\u2300-\u23FF\u2B00-\u2BFF\uFE0F\u200D]+", flags=re.UNICODE)
|
|
47
47
|
ARABIZI_CANDIDATE_REGEX = re.compile(r"\b([A-Za-z][A-Za-z0-9]*|[2356789][A-Za-z]{2,}[0-9]*)\b")
|
|
48
48
|
SUPPORTED_EMOJI_MODES = ("strip", "keep", "sentiment")
|
|
49
49
|
|
|
@@ -193,14 +193,21 @@ class MSALightPipeline:
|
|
|
193
193
|
preserving Transformer-native orthography (أ/إ/آ, ة, ى) by default.
|
|
194
194
|
"""
|
|
195
195
|
|
|
196
|
-
def __init__(
|
|
196
|
+
def __init__(
|
|
197
|
+
self,
|
|
198
|
+
preserve_orthography: bool = True,
|
|
199
|
+
normalize_digits: bool = False,
|
|
200
|
+
emoji_mode: str = "strip",
|
|
201
|
+
):
|
|
197
202
|
self.preserve_orthography = preserve_orthography
|
|
198
203
|
self.normalize_digits = normalize_digits
|
|
204
|
+
self.emoji_mode = emoji_mode
|
|
199
205
|
|
|
200
206
|
def process_with_stats(self, text: str) -> Tuple[str, int, int]:
|
|
201
207
|
t = strip_markup_noise(text)
|
|
202
208
|
t = ZERO_WIDTH_REGEX.sub("", t)
|
|
203
209
|
t, clitic_fixed = clean_code_switching_clitics(t)
|
|
210
|
+
t = process_emojis(t, self.emoji_mode)
|
|
204
211
|
t = TASHKEEL_REGEX.sub("", t)
|
|
205
212
|
t = TATWEEL_REGEX.sub("", t)
|
|
206
213
|
if self.preserve_orthography:
|
|
@@ -109,7 +109,10 @@ class MisarRouter:
|
|
|
109
109
|
self.emoji_mode = emoji_mode
|
|
110
110
|
self.arabizi_mode = arabizi_mode
|
|
111
111
|
self.profiler = LinguisticProfiler()
|
|
112
|
-
self.msa_pipeline = MSALightPipeline(
|
|
112
|
+
self.msa_pipeline = MSALightPipeline(
|
|
113
|
+
preserve_orthography=preserve_orthography,
|
|
114
|
+
emoji_mode=emoji_mode,
|
|
115
|
+
)
|
|
113
116
|
self.dialect_pipeline = DialectPreservePipeline(
|
|
114
117
|
preserve_orthography=preserve_orthography,
|
|
115
118
|
tokenizer=self.tokenizer,
|
|
@@ -192,9 +195,6 @@ class MisarRouter:
|
|
|
192
195
|
else:
|
|
193
196
|
reason = "نص فصحى معياري نقي تم إعفاؤه من التحليل الصرفي الثقيل مع صون الهمزات والتاء المربوطة"
|
|
194
197
|
processed, healed, clitic_fixed = self.msa_pipeline.process_with_stats(text)
|
|
195
|
-
if self.emoji_mode != "strip":
|
|
196
|
-
from .pipelines import MULTISPACE_REGEX, process_emojis
|
|
197
|
-
processed = MULTISPACE_REGEX.sub(" ", process_emojis(processed, self.emoji_mode)).strip()
|
|
198
198
|
return route_name, route_display, reason, processed, healed, clitic_fixed
|
|
199
199
|
|
|
200
200
|
def preprocess(self, text: str) -> str:
|
|
@@ -654,7 +654,7 @@
|
|
|
654
654
|
<header class="topbar">
|
|
655
655
|
<div class="topbar-inner">
|
|
656
656
|
<div class="brand">
|
|
657
|
-
<h1>مِسار — Misar <span class="version-tag ltr">v2.0.
|
|
657
|
+
<h1>مِسار — Misar <span class="version-tag ltr">v2.0.1</span></h1>
|
|
658
658
|
<p>موجه المعالجة العربية التكيفية لتعزيز كفاءة نماذج المحولات اللغوية (<span class="ltr">Adaptive Arabic NLP Router</span>)</p>
|
|
659
659
|
</div>
|
|
660
660
|
<div class="status-group">
|
|
@@ -872,7 +872,7 @@
|
|
|
872
872
|
</section>
|
|
873
873
|
|
|
874
874
|
<footer class="page-footer ltr">
|
|
875
|
-
Misar (مِسار) v2.0.
|
|
875
|
+
Misar (مِسار) v2.0.1 · Open-Source MIT License · Copyright © 2026 Muneer Jarab Matrak Al-Theeb
|
|
876
876
|
</footer>
|
|
877
877
|
</main>
|
|
878
878
|
|
|
@@ -950,7 +950,7 @@
|
|
|
950
950
|
const RE_ZERO_WIDTH = /[\u200B-\u200F\u202A-\u202E\u2060\uFEFF]/g;
|
|
951
951
|
const RE_MARRA = /^[وفب]?مر{2,}[ةه]$/;
|
|
952
952
|
const RE_REPEATED = /(\p{L})\1{2,}/gu;
|
|
953
|
-
const RE_EMOJI = /[\u{10000}-\u{10FFFF}\u2600-\u27BF\uFE0F]/gu;
|
|
953
|
+
const RE_EMOJI = /[\u{10000}-\u{10FFFF}\u2600-\u27BF\u2300-\u23FF\u2B00-\u2BFF\uFE0F]/gu;
|
|
954
954
|
const RE_PUNCT_REPEAT = /([!?.،؛؟]){2,}/g;
|
|
955
955
|
const RE_HASHTAG = /#([\p{L}\p{N}_]+)/gu;
|
|
956
956
|
const RE_URL = /https?:\/\/\S+|www\.\S+/gi;
|
|
@@ -1066,8 +1066,8 @@
|
|
|
1066
1066
|
}
|
|
1067
1067
|
|
|
1068
1068
|
function processEmojiJS(text, mode) {
|
|
1069
|
-
if (mode === "keep") return text.replace(/[\u{10000}-\u{10FFFF}\u2600-\u27BF\uFE0F\u200D]+/gu, m => { const d = [...m].filter(c => c !== "\uFE0F" && c !== "\u200D"); const u = d.filter((c, i) => i === 0 || c !== d[i - 1]); return u.length ? ` ${u.join("")} ` : " "; });
|
|
1070
|
-
if (mode === "sentiment") return text.replace(/[\u{10000}-\u{10FFFF}\u2600-\u27BF\uFE0F\u200D]+/gu, m => { let pos = 0, neg = 0; for (const c of m) { if (JS_POS_EMOJI.has(c)) pos++; if (JS_NEG_EMOJI.has(c)) neg++; } return neg > pos ? " سلبي " : (pos > neg ? " إيجابي " : " "); }).replace(/\b(سلبي|إيجابي)(\s+\1)+/g, "$1");
|
|
1069
|
+
if (mode === "keep") return text.replace(/[\u{10000}-\u{10FFFF}\u2600-\u27BF\u2300-\u23FF\u2B00-\u2BFF\uFE0F\u200D]+/gu, m => { const d = [...m].filter(c => c !== "\uFE0F" && c !== "\u200D"); const u = d.filter((c, i) => i === 0 || c !== d[i - 1]); return u.length ? ` ${u.join("")} ` : " "; });
|
|
1070
|
+
if (mode === "sentiment") return text.replace(/[\u{10000}-\u{10FFFF}\u2600-\u27BF\u2300-\u23FF\u2B00-\u2BFF\uFE0F\u200D]+/gu, m => { let pos = 0, neg = 0; for (const c of m) { if (JS_POS_EMOJI.has(c)) pos++; if (JS_NEG_EMOJI.has(c)) neg++; } return neg > pos ? " سلبي " : (pos > neg ? " إيجابي " : " "); }).replace(/\b(سلبي|إيجابي)(\s+\1)+/g, "$1");
|
|
1071
1071
|
return text.replace(RE_EMOJI, " ");
|
|
1072
1072
|
}
|
|
1073
1073
|
|
|
@@ -1363,21 +1363,44 @@
|
|
|
1363
1363
|
barRed.style.width = `${Math.min(100, Math.round(redVal))}%`;
|
|
1364
1364
|
barRed.style.backgroundColor = "";
|
|
1365
1365
|
const sign = data.token_reduction_pct >= 0 ? "-" : "+";
|
|
1366
|
-
txtRed.textContent =
|
|
1366
|
+
txtRed.textContent = "خفض الرموز وتوفير التكلفة: ";
|
|
1367
|
+
const redSpan = document.createElement("span");
|
|
1368
|
+
redSpan.className = "ltr";
|
|
1369
|
+
redSpan.textContent = `${sign}${Math.abs(data.token_reduction_pct).toFixed(1)}%`;
|
|
1370
|
+
txtRed.appendChild(redSpan);
|
|
1371
|
+
|
|
1372
|
+
let legacyText = data.legacy_text;
|
|
1373
|
+
let legacyTokens = data.legacy_tokens;
|
|
1374
|
+
let legacyCount = data.legacy_token_count;
|
|
1375
|
+
let legacyTfr = data.legacy_tfr;
|
|
1376
|
+
let vsLeg = data.vs_legacy_reduction_pct;
|
|
1377
|
+
if (legacyText === undefined) {
|
|
1378
|
+
const fb = routeOfflineJS(data.original_text || "");
|
|
1379
|
+
legacyText = fb.legacy_text;
|
|
1380
|
+
legacyTokens = fb.legacy_tokens;
|
|
1381
|
+
legacyCount = fb.legacy_token_count;
|
|
1382
|
+
legacyTfr = fb.legacy_tfr;
|
|
1383
|
+
vsLeg = legacyCount ? ((legacyCount - data.processed_token_count) / legacyCount) * 100 : 0;
|
|
1384
|
+
}
|
|
1367
1385
|
|
|
1368
1386
|
document.getElementById("rawTextDisplay").textContent = data.original_text || "—";
|
|
1369
|
-
document.getElementById("legacyTextDisplay").textContent =
|
|
1387
|
+
document.getElementById("legacyTextDisplay").textContent = legacyText || "—";
|
|
1370
1388
|
document.getElementById("procTextDisplay").textContent = data.processed_text || "—";
|
|
1371
1389
|
document.getElementById("rawTokensCount").textContent = `${data.raw_token_count} Tokens · TFR ${data.tfr_raw.toFixed(2)}`;
|
|
1372
|
-
document.getElementById("legacyTokensCount").textContent = `${
|
|
1390
|
+
document.getElementById("legacyTokensCount").textContent = `${legacyCount || 0} Tokens · TFR ${(legacyTfr || 0).toFixed(2)}`;
|
|
1373
1391
|
document.getElementById("procTokensCount").textContent = `${data.processed_token_count} Tokens · TFR ${data.tfr_processed.toFixed(2)}`;
|
|
1374
1392
|
|
|
1375
|
-
const
|
|
1376
|
-
const
|
|
1377
|
-
|
|
1393
|
+
const legSign = (vsLeg || 0) >= 0 ? "-" : "+";
|
|
1394
|
+
const advEl = document.getElementById("v2AdvantageBadge");
|
|
1395
|
+
advEl.textContent = "تفوق على التقليدي: ";
|
|
1396
|
+
const advPct = document.createElement("span");
|
|
1397
|
+
advPct.className = "ltr";
|
|
1398
|
+
advPct.textContent = `${legSign}${Math.abs(vsLeg || 0).toFixed(1)}%`;
|
|
1399
|
+
advEl.appendChild(advPct);
|
|
1400
|
+
advEl.appendChild(document.createTextNode(` | ترميم إملائي: ${data.auto_healed_count || 0} | صلة لغوية: ${data.clitic_fixed_count || 0}`));
|
|
1378
1401
|
|
|
1379
1402
|
renderChips("rawTokensChips", data.raw_tokens, true);
|
|
1380
|
-
renderChips("legacyTokensChips",
|
|
1403
|
+
renderChips("legacyTokensChips", legacyTokens || [], true);
|
|
1381
1404
|
renderChips("procTokensChips", data.processed_tokens, false);
|
|
1382
1405
|
}
|
|
1383
1406
|
|
|
@@ -244,6 +244,18 @@ class TestMisarRoutingAndPipelines(unittest.TestCase):
|
|
|
244
244
|
self.assertTrue(args.compare)
|
|
245
245
|
self.assertTrue(args.offline)
|
|
246
246
|
|
|
247
|
+
def test_fast_msa_long_diacritized_essay_strips_all_emojis(self):
|
|
248
|
+
essay = (
|
|
249
|
+
"🌍 ✨ فِي هَذِهِ الحَيَاةِ الوَاسِعَةِ، تَتَعَاقَبُ الأَيَّامُ وَتَمْضِي السَّاعَاتُ، وَيَبْقَى الإِنْسَانُ يَسْعَى نَحْوَ أَحْلَامِهِ "
|
|
250
|
+
"حَامِلاً فِي قَلْبِهِ أَمَلاً جَدِيداً. ⏳ ⭐ إِنَّ العِلْمَ نُورٌ يُضِيءُ دُرُوبَ المُسْتَقْبَلِ، وَالمَعْرِفَةُ كَنْزٌ لَا يَنْفَدُ، "
|
|
251
|
+
"وَالْعَمَلُ الطَّرِيقُ الحَقِيقِيُّ لِبُلُوغِ النَّجَاحِ وَتَحْقِيقِ الطُّمُوحِ."
|
|
252
|
+
)
|
|
253
|
+
res = self.router_offline.route(essay)
|
|
254
|
+
self.assertEqual(res.route_name, "FAST_MSA")
|
|
255
|
+
for em in ("🌍", "✨", "⏳", "⭐"):
|
|
256
|
+
self.assertNotIn(em, res.processed_text)
|
|
257
|
+
self.assertEqual(res.unk_count_processed, 0)
|
|
258
|
+
|
|
247
259
|
|
|
248
260
|
class TestRealAraBERTv2WordPiece(unittest.TestCase):
|
|
249
261
|
"""Empirical proof against the authentic 64,000-token AraBERTv2 vocabulary."""
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|