misar 2.0.0__tar.gz → 2.0.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: misar
3
- Version: 2.0.0
3
+ Version: 2.0.1
4
4
  Summary: Misar (مِسار): Adaptive context-aware Arabic NLP preprocessing router for optimizing Transformer language models.
5
5
  Author: Muneer Jarab Matrak Al-Theeb
6
6
  License: MIT
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "misar"
7
- version = "2.0.0"
7
+ version = "2.0.1"
8
8
  description = "Misar (مِسار): Adaptive context-aware Arabic NLP preprocessing router for optimizing Transformer language models."
9
9
  readme = "README.md"
10
10
  license = {text = "MIT"}
@@ -15,7 +15,7 @@ from .tokenizer import (
15
15
  load_default_tokenizer,
16
16
  )
17
17
 
18
- __version__ = "2.0.0"
18
+ __version__ = "2.0.1"
19
19
  __author__ = "Muneer Jarab Matrak Al-Theeb"
20
20
 
21
21
  __all__ = [
@@ -11,7 +11,7 @@ TATWEEL_REGEX = re.compile(r"\u0640")
11
11
  DECORATION_MARKS_REGEX = re.compile(r"[\u0640\u064B-\u065F\u0670]")
12
12
  REPEATED_CHARS_REGEX = re.compile(r"([^\s\d\W_])\1{2,}")
13
13
  EMOJI_REGEX = re.compile(
14
- r"[\U00010000-\U0010FFFF\u2600-\u27BF\uFE0F]",
14
+ r"[\U00010000-\U0010FFFF\u2600-\u27BF\u2300-\u23FF\u2B00-\u2BFF\uFE0F]",
15
15
  flags=re.UNICODE,
16
16
  )
17
17
  PUNCT_REPEAT_REGEX = re.compile(r"([!?.،؛؟]){2,}")
@@ -39,7 +39,7 @@ POSSESSIVE_AMBIGUOUS_H_WORDS: Set[str] = {
39
39
 
40
40
  POSITIVE_EMOJIS: Set[str] = {
41
41
  "😍", "🥰", "😘", "😊", "😁", "😄", "😃", "🥳", "🤩", "🙌", "👏",
42
- "👍", "👌", "🔥", "🚀", "✨", "🌟", "💯", "✅", "♥", "❤", "💖", "💙", "💚",
42
+ "👍", "👌", "🔥", "🚀", "✨", "🌟", "⭐", "💯", "✅", "♥", "❤", "💖", "💙", "💚",
43
43
  }
44
44
 
45
45
  NEGATIVE_EMOJIS: Set[str] = {
@@ -43,7 +43,7 @@ MULTISPACE_REGEX = re.compile(r"\s+")
43
43
  MARKDOWN_HEADER_REGEX = re.compile(r"(^|\n)\s*#{1,6}\s+")
44
44
  MARKDOWN_FORMAT_REGEX = re.compile(r"[*~`]")
45
45
  ARABIC_FINAL_HAA_WORD_REGEX = re.compile(r"\b([\u0621-\u064A]{4,})\b")
46
- EMOJI_RUN_REGEX = re.compile(r"[\U00010000-\U0010FFFF\u2600-\u27BF\uFE0F\u200D]+", flags=re.UNICODE)
46
+ EMOJI_RUN_REGEX = re.compile(r"[\U00010000-\U0010FFFF\u2600-\u27BF\u2300-\u23FF\u2B00-\u2BFF\uFE0F\u200D]+", flags=re.UNICODE)
47
47
  ARABIZI_CANDIDATE_REGEX = re.compile(r"\b([A-Za-z][A-Za-z0-9]*|[2356789][A-Za-z]{2,}[0-9]*)\b")
48
48
  SUPPORTED_EMOJI_MODES = ("strip", "keep", "sentiment")
49
49
 
@@ -193,14 +193,21 @@ class MSALightPipeline:
193
193
  preserving Transformer-native orthography (أ/إ/آ, ة, ى) by default.
194
194
  """
195
195
 
196
- def __init__(self, preserve_orthography: bool = True, normalize_digits: bool = False):
196
+ def __init__(
197
+ self,
198
+ preserve_orthography: bool = True,
199
+ normalize_digits: bool = False,
200
+ emoji_mode: str = "strip",
201
+ ):
197
202
  self.preserve_orthography = preserve_orthography
198
203
  self.normalize_digits = normalize_digits
204
+ self.emoji_mode = emoji_mode
199
205
 
200
206
  def process_with_stats(self, text: str) -> Tuple[str, int, int]:
201
207
  t = strip_markup_noise(text)
202
208
  t = ZERO_WIDTH_REGEX.sub("", t)
203
209
  t, clitic_fixed = clean_code_switching_clitics(t)
210
+ t = process_emojis(t, self.emoji_mode)
204
211
  t = TASHKEEL_REGEX.sub("", t)
205
212
  t = TATWEEL_REGEX.sub("", t)
206
213
  if self.preserve_orthography:
@@ -109,7 +109,10 @@ class MisarRouter:
109
109
  self.emoji_mode = emoji_mode
110
110
  self.arabizi_mode = arabizi_mode
111
111
  self.profiler = LinguisticProfiler()
112
- self.msa_pipeline = MSALightPipeline(preserve_orthography=preserve_orthography)
112
+ self.msa_pipeline = MSALightPipeline(
113
+ preserve_orthography=preserve_orthography,
114
+ emoji_mode=emoji_mode,
115
+ )
113
116
  self.dialect_pipeline = DialectPreservePipeline(
114
117
  preserve_orthography=preserve_orthography,
115
118
  tokenizer=self.tokenizer,
@@ -192,9 +195,6 @@ class MisarRouter:
192
195
  else:
193
196
  reason = "نص فصحى معياري نقي تم إعفاؤه من التحليل الصرفي الثقيل مع صون الهمزات والتاء المربوطة"
194
197
  processed, healed, clitic_fixed = self.msa_pipeline.process_with_stats(text)
195
- if self.emoji_mode != "strip":
196
- from .pipelines import MULTISPACE_REGEX, process_emojis
197
- processed = MULTISPACE_REGEX.sub(" ", process_emojis(processed, self.emoji_mode)).strip()
198
198
  return route_name, route_display, reason, processed, healed, clitic_fixed
199
199
 
200
200
  def preprocess(self, text: str) -> str:
@@ -654,7 +654,7 @@
654
654
  <header class="topbar">
655
655
  <div class="topbar-inner">
656
656
  <div class="brand">
657
- <h1>مِسار — Misar <span class="version-tag ltr">v2.0.0</span></h1>
657
+ <h1>مِسار — Misar <span class="version-tag ltr">v2.0.1</span></h1>
658
658
  <p>موجه المعالجة العربية التكيفية لتعزيز كفاءة نماذج المحولات اللغوية (<span class="ltr">Adaptive Arabic NLP Router</span>)</p>
659
659
  </div>
660
660
  <div class="status-group">
@@ -872,7 +872,7 @@
872
872
  </section>
873
873
 
874
874
  <footer class="page-footer ltr">
875
- Misar (مِسار) v2.0.0 · Open-Source MIT License · Copyright © 2026 Muneer Jarab Matrak Al-Theeb
875
+ Misar (مِسار) v2.0.1 · Open-Source MIT License · Copyright © 2026 Muneer Jarab Matrak Al-Theeb
876
876
  </footer>
877
877
  </main>
878
878
 
@@ -950,7 +950,7 @@
950
950
  const RE_ZERO_WIDTH = /[\u200B-\u200F\u202A-\u202E\u2060\uFEFF]/g;
951
951
  const RE_MARRA = /^[وفب]?مر{2,}[ةه]$/;
952
952
  const RE_REPEATED = /(\p{L})\1{2,}/gu;
953
- const RE_EMOJI = /[\u{10000}-\u{10FFFF}\u2600-\u27BF\uFE0F]/gu;
953
+ const RE_EMOJI = /[\u{10000}-\u{10FFFF}\u2600-\u27BF\u2300-\u23FF\u2B00-\u2BFF\uFE0F]/gu;
954
954
  const RE_PUNCT_REPEAT = /([!?.،؛؟]){2,}/g;
955
955
  const RE_HASHTAG = /#([\p{L}\p{N}_]+)/gu;
956
956
  const RE_URL = /https?:\/\/\S+|www\.\S+/gi;
@@ -1066,8 +1066,8 @@
1066
1066
  }
1067
1067
 
1068
1068
  function processEmojiJS(text, mode) {
1069
- if (mode === "keep") return text.replace(/[\u{10000}-\u{10FFFF}\u2600-\u27BF\uFE0F\u200D]+/gu, m => { const d = [...m].filter(c => c !== "\uFE0F" && c !== "\u200D"); const u = d.filter((c, i) => i === 0 || c !== d[i - 1]); return u.length ? ` ${u.join("")} ` : " "; });
1070
- if (mode === "sentiment") return text.replace(/[\u{10000}-\u{10FFFF}\u2600-\u27BF\uFE0F\u200D]+/gu, m => { let pos = 0, neg = 0; for (const c of m) { if (JS_POS_EMOJI.has(c)) pos++; if (JS_NEG_EMOJI.has(c)) neg++; } return neg > pos ? " سلبي " : (pos > neg ? " إيجابي " : " "); }).replace(/\b(سلبي|إيجابي)(\s+\1)+/g, "$1");
1069
+ if (mode === "keep") return text.replace(/[\u{10000}-\u{10FFFF}\u2600-\u27BF\u2300-\u23FF\u2B00-\u2BFF\uFE0F\u200D]+/gu, m => { const d = [...m].filter(c => c !== "\uFE0F" && c !== "\u200D"); const u = d.filter((c, i) => i === 0 || c !== d[i - 1]); return u.length ? ` ${u.join("")} ` : " "; });
1070
+ if (mode === "sentiment") return text.replace(/[\u{10000}-\u{10FFFF}\u2600-\u27BF\u2300-\u23FF\u2B00-\u2BFF\uFE0F\u200D]+/gu, m => { let pos = 0, neg = 0; for (const c of m) { if (JS_POS_EMOJI.has(c)) pos++; if (JS_NEG_EMOJI.has(c)) neg++; } return neg > pos ? " سلبي " : (pos > neg ? " إيجابي " : " "); }).replace(/\b(سلبي|إيجابي)(\s+\1)+/g, "$1");
1071
1071
  return text.replace(RE_EMOJI, " ");
1072
1072
  }
1073
1073
 
@@ -1363,21 +1363,44 @@
1363
1363
  barRed.style.width = `${Math.min(100, Math.round(redVal))}%`;
1364
1364
  barRed.style.backgroundColor = "";
1365
1365
  const sign = data.token_reduction_pct >= 0 ? "-" : "+";
1366
- txtRed.textContent = `خفض الرموز وتوفير التكلفة: ${sign}${Math.abs(data.token_reduction_pct).toFixed(1)}%`;
1366
+ txtRed.textContent = "خفض الرموز وتوفير التكلفة: ";
1367
+ const redSpan = document.createElement("span");
1368
+ redSpan.className = "ltr";
1369
+ redSpan.textContent = `${sign}${Math.abs(data.token_reduction_pct).toFixed(1)}%`;
1370
+ txtRed.appendChild(redSpan);
1371
+
1372
+ let legacyText = data.legacy_text;
1373
+ let legacyTokens = data.legacy_tokens;
1374
+ let legacyCount = data.legacy_token_count;
1375
+ let legacyTfr = data.legacy_tfr;
1376
+ let vsLeg = data.vs_legacy_reduction_pct;
1377
+ if (legacyText === undefined) {
1378
+ const fb = routeOfflineJS(data.original_text || "");
1379
+ legacyText = fb.legacy_text;
1380
+ legacyTokens = fb.legacy_tokens;
1381
+ legacyCount = fb.legacy_token_count;
1382
+ legacyTfr = fb.legacy_tfr;
1383
+ vsLeg = legacyCount ? ((legacyCount - data.processed_token_count) / legacyCount) * 100 : 0;
1384
+ }
1367
1385
 
1368
1386
  document.getElementById("rawTextDisplay").textContent = data.original_text || "—";
1369
- document.getElementById("legacyTextDisplay").textContent = data.legacy_text || "—";
1387
+ document.getElementById("legacyTextDisplay").textContent = legacyText || "—";
1370
1388
  document.getElementById("procTextDisplay").textContent = data.processed_text || "—";
1371
1389
  document.getElementById("rawTokensCount").textContent = `${data.raw_token_count} Tokens · TFR ${data.tfr_raw.toFixed(2)}`;
1372
- document.getElementById("legacyTokensCount").textContent = `${data.legacy_token_count || 0} Tokens · TFR ${(data.legacy_tfr || 0).toFixed(2)}`;
1390
+ document.getElementById("legacyTokensCount").textContent = `${legacyCount || 0} Tokens · TFR ${(legacyTfr || 0).toFixed(2)}`;
1373
1391
  document.getElementById("procTokensCount").textContent = `${data.processed_token_count} Tokens · TFR ${data.tfr_processed.toFixed(2)}`;
1374
1392
 
1375
- const vsLeg = data.vs_legacy_reduction_pct || 0;
1376
- const legSign = vsLeg >= 0 ? "-" : "+";
1377
- document.getElementById("v2AdvantageBadge").textContent = `تفوق على التقليدي: ${legSign}${Math.abs(vsLeg).toFixed(1)}% | ترميم إملائي: ${data.auto_healed_count || 0} | صلة لغوية: ${data.clitic_fixed_count || 0}`;
1393
+ const legSign = (vsLeg || 0) >= 0 ? "-" : "+";
1394
+ const advEl = document.getElementById("v2AdvantageBadge");
1395
+ advEl.textContent = "تفوق على التقليدي: ";
1396
+ const advPct = document.createElement("span");
1397
+ advPct.className = "ltr";
1398
+ advPct.textContent = `${legSign}${Math.abs(vsLeg || 0).toFixed(1)}%`;
1399
+ advEl.appendChild(advPct);
1400
+ advEl.appendChild(document.createTextNode(` | ترميم إملائي: ${data.auto_healed_count || 0} | صلة لغوية: ${data.clitic_fixed_count || 0}`));
1378
1401
 
1379
1402
  renderChips("rawTokensChips", data.raw_tokens, true);
1380
- renderChips("legacyTokensChips", data.legacy_tokens || [], true);
1403
+ renderChips("legacyTokensChips", legacyTokens || [], true);
1381
1404
  renderChips("procTokensChips", data.processed_tokens, false);
1382
1405
  }
1383
1406
 
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: misar
3
- Version: 2.0.0
3
+ Version: 2.0.1
4
4
  Summary: Misar (مِسار): Adaptive context-aware Arabic NLP preprocessing router for optimizing Transformer language models.
5
5
  Author: Muneer Jarab Matrak Al-Theeb
6
6
  License: MIT
@@ -244,6 +244,18 @@ class TestMisarRoutingAndPipelines(unittest.TestCase):
244
244
  self.assertTrue(args.compare)
245
245
  self.assertTrue(args.offline)
246
246
 
247
+ def test_fast_msa_long_diacritized_essay_strips_all_emojis(self):
248
+ essay = (
249
+ "🌍 ✨ فِي هَذِهِ الحَيَاةِ الوَاسِعَةِ، تَتَعَاقَبُ الأَيَّامُ وَتَمْضِي السَّاعَاتُ، وَيَبْقَى الإِنْسَانُ يَسْعَى نَحْوَ أَحْلَامِهِ "
250
+ "حَامِلاً فِي قَلْبِهِ أَمَلاً جَدِيداً. ⏳ ⭐ إِنَّ العِلْمَ نُورٌ يُضِيءُ دُرُوبَ المُسْتَقْبَلِ، وَالمَعْرِفَةُ كَنْزٌ لَا يَنْفَدُ، "
251
+ "وَالْعَمَلُ الطَّرِيقُ الحَقِيقِيُّ لِبُلُوغِ النَّجَاحِ وَتَحْقِيقِ الطُّمُوحِ."
252
+ )
253
+ res = self.router_offline.route(essay)
254
+ self.assertEqual(res.route_name, "FAST_MSA")
255
+ for em in ("🌍", "✨", "⏳", "⭐"):
256
+ self.assertNotIn(em, res.processed_text)
257
+ self.assertEqual(res.unk_count_processed, 0)
258
+
247
259
 
248
260
  class TestRealAraBERTv2WordPiece(unittest.TestCase):
249
261
  """Empirical proof against the authentic 64,000-token AraBERTv2 vocabulary."""
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes