versed-pdf 1.1.0__py3-none-any.whl

versed/__init__.py

@@ -0,0 +1,144 @@
+"""versed — local PDF-to-Markdown tooling for Arabic and bilingual texts."""
+
+__version__ = "1.1.0"
+
+from .arabic import (
+    detect_batch_reversal,
+    is_arabic,
+    is_mostly_arabic,
+    orphan_diacritic_rate,
+    strip_diacritics,
+)
+from .classify import (
+    BackendConfig,
+    PageProbe,
+    PageType,
+    classify_and_select,
+    classify_page,
+    select_backend,
+)
+from .detect import (
+    KNOWN_MOJIBAKE_CHARS,
+    MojibakeReport,
+    detect_mojibake,
+    detect_mojibake_in_pdf,
+)
+from .extract import ExtractResult, extract_document
+from .health import summarize_text_health
+from .honorifics import (
+    HONORIFIC_SYMBOLS,
+    NormalizedWord,
+    annotate_transliterations,
+    decode_honorific,
+    expand_honorifics,
+    find_transliteration,
+    get_spoken_text,
+    has_leading_honorific,
+    normalize_text,
+    normalize_words,
+)
+from .layout import (
+    document_from_aligned_words,
+    document_from_markdown,
+    document_from_structured,
+)
+from .markdown import (
+    EnhancedMarkdownResult,
+    build_enhanced_markdown,
+    compute_aligned_words_checksum,
+    compute_cache_key,
+)
+from .qcf import (
+    QCFDecoder,
+    QCFVerse,
+    QCFWord,
+    build_qcf_mapping_from_quran_data,
+    detect_qcf_regions,
+    extract_qcf_page_number,
+    is_qcf_glyph,
+    is_qcf_text,
+)
+from .repair import (
+    SABON_CHAR_REPAIR,
+    SABON_FONT_PREFIXES,
+    extract_repairable_font_spans,
+    find_font_for_word,
+    is_repairable_font,
+    repair_text,
+    repair_text_for_font,
+    repair_words_with_font_info,
+)
+from .routing import (
+    EnrichmentDecision,
+    PageObservations,
+    TaskNeeds,
+    observe_from_extraction,
+    observe_page,
+    route_enrichment,
+)
+from .types import AlignedWord, BlockType, Document, TextBlock, WordBox
+
+__all__ = [
+    "AlignedWord",
+    "BackendConfig",
+    "BlockType",
+    "Document",
+    "EnhancedMarkdownResult",
+    "EnrichmentDecision",
+    "ExtractResult",
+    "HONORIFIC_SYMBOLS",
+    "KNOWN_MOJIBAKE_CHARS",
+    "MojibakeReport",
+    "NormalizedWord",
+    "PageObservations",
+    "PageProbe",
+    "PageType",
+    "QCFDecoder",
+    "QCFVerse",
+    "QCFWord",
+    "SABON_CHAR_REPAIR",
+    "SABON_FONT_PREFIXES",
+    "TaskNeeds",
+    "TextBlock",
+    "WordBox",
+    "annotate_transliterations",
+    "build_enhanced_markdown",
+    "build_qcf_mapping_from_quran_data",
+    "classify_and_select",
+    "classify_page",
+    "compute_aligned_words_checksum",
+    "compute_cache_key",
+    "decode_honorific",
+    "detect_batch_reversal",
+    "detect_mojibake",
+    "detect_mojibake_in_pdf",
+    "detect_qcf_regions",
+    "document_from_aligned_words",
+    "document_from_markdown",
+    "document_from_structured",
+    "expand_honorifics",
+    "extract_document",
+    "extract_qcf_page_number",
+    "extract_repairable_font_spans",
+    "find_font_for_word",
+    "find_transliteration",
+    "get_spoken_text",
+    "has_leading_honorific",
+    "is_arabic",
+    "is_mostly_arabic",
+    "is_qcf_glyph",
+    "is_qcf_text",
+    "is_repairable_font",
+    "normalize_text",
+    "normalize_words",
+    "observe_from_extraction",
+    "observe_page",
+    "orphan_diacritic_rate",
+    "repair_text",
+    "repair_text_for_font",
+    "repair_words_with_font_info",
+    "route_enrichment",
+    "select_backend",
+    "strip_diacritics",
+    "summarize_text_health",
+]

versed/_arabic.py

@@ -0,0 +1,213 @@
+"""
+Arabic text utilities — normalization, detection, similarity.
+
+Internal module used by honorifics.py and detect.py.
+Not part of the public API.
+"""
+
+import re
+import unicodedata
+from typing import Dict, Optional
+
+
+class TextUtils:
+    """Text processing utilities."""
+
+    # Arabic character ranges
+    ARABIC_PATTERN = re.compile(r'[\u0600-\u06FF\u0750-\u077F\uFB50-\uFDFF\uFE70-\uFEFF]')
+
+    # Punctuation and diacritics to strip
+    PUNCTUATION_PATTERN = re.compile(
+        r"[\'\"`´′″‹›«»\u2018\u2019\u201C\u201D\u02BC\u2032\u2033.,;:!?()[\]{}،؛؟–—-]"
+    )
+    DIACRITICS_PATTERN = re.compile(r'[\u064B-\u065F\u0670]')
+
+    # Alef variants (أإآٱ) → bare alef (ا)
+    ALEF_VARIANTS = re.compile(r'[أإآٱ]')
+    # Zero-width and directional markers to drop
+    ZERO_WIDTH_TRANSLATION = str.maketrans({
+        "\u200B": "",  # zero width space
+        "\u200C": "",  # zero width non-joiner
+        "\u200D": "",  # zero width joiner
+        "\u2060": "",  # word joiner
+        "\uFEFF": "",  # zero width no-break space
+        "\u200E": "",  # LRM
+        "\u200F": "",  # RLM
+    })
+    # Common Latin ligatures (extra safety; NFKC should also handle these)
+    LIGATURE_TRANSLATION = str.maketrans({
+        "\uFB00": "ff",
+        "\uFB01": "fi",
+        "\uFB02": "fl",
+        "\uFB03": "ffi",
+        "\uFB04": "ffl",
+        "\uFB05": "st",
+        "\uFB06": "st",
+    })
+
+    @classmethod
+    def normalize_arabic(cls, text: str) -> str:
+        """
+        Normalize Arabic text for comparison/matching.
+
+        - Removes diacritics (tashkeel)
+        - Normalizes alef variants to bare alef
+        - Normalizes teh marbuta to heh
+        - Normalizes alef maksura to yeh
+        - Removes tatweel (kashida)
+        """
+        if not text:
+            return ""
+
+        # Remove tashkeel (Arabic diacritics)
+        text = cls.DIACRITICS_PATTERN.sub('', text)
+
+        # Normalize alef variants to bare alef
+        text = cls.ALEF_VARIANTS.sub('ا', text)
+
+        # Normalize teh marbuta to heh
+        text = text.replace('ة', 'ه')
+
+        # Normalize alef maksura to yeh
+        text = text.replace('ى', 'ي')
+
+        # Remove tatweel (kashida)
+        text = text.replace('\u0640', '')
+
+        return text.strip()
+
+    @classmethod
+    def normalize(cls, text: str) -> str:
+        """
+        Normalize text for matching.
+
+        - Lowercases (for non-Arabic)
+        - Strips whitespace
+        - Removes punctuation
+        - Applies full Arabic normalization (diacritics, alef variants, etc.)
+        """
+        if not text:
+            return ""
+        text = unicodedata.normalize("NFKC", text).strip()
+        text = text.translate(cls.ZERO_WIDTH_TRANSLATION)
+        text = text.translate(cls.LIGATURE_TRANSLATION)
+        text = cls.PUNCTUATION_PATTERN.sub("", text)
+
+        # Apply full Arabic normalization
+        text = cls.normalize_arabic(text)
+
+        # Lowercase for non-Arabic matching (safe after Arabic normalization)
+        text = text.lower()
+
+        return text.strip()
+
+    @classmethod
+    def is_arabic(cls, text: str) -> bool:
+        """Check if text contains Arabic characters."""
+        return bool(cls.ARABIC_PATTERN.search(text)) if text else False
+
+    @classmethod
+    def levenshtein_similarity(cls, s1: str, s2: str) -> float:
+        """
+        Calculate similarity score between two strings.
+
+        Returns:
+            Similarity score 0.0-1.0 (1.0 = identical)
+        """
+        s1, s2 = cls.normalize(s1), cls.normalize(s2)
+        if not s1 or not s2:
+            return 0.0
+        if s1 == s2:
+            return 1.0
+
+        len1, len2 = len(s1), len(s2)
+        if len1 > len2:
+            s1, s2 = s2, s1
+            len1, len2 = len2, len1
+
+        current_row = list(range(len1 + 1))
+        for i in range(1, len2 + 1):
+            previous_row, current_row = current_row, [i] + [0] * len1
+            for j in range(1, len1 + 1):
+                add = previous_row[j] + 1
+                delete = current_row[j - 1] + 1
+                change = previous_row[j - 1] + (0 if s1[j - 1] == s2[i - 1] else 1)
+                current_row[j] = min(add, delete, change)
+
+        distance = current_row[len1]
+        return 1.0 - (distance / max(len1, len2))
+
+
+class HonorificsHandler:
+    """
+    Handler for Islamic honorific symbols.
+
+    Maps display symbols (ﷺ, ﷻ, etc.) to spoken Arabic phrases.
+    """
+
+    # Honorific expansions WITH HARAKAT for proper TTS pronunciation
+    HONORIFICS: Dict[str, str] = {
+        # Prophet Muhammad ﷺ - صَلَّى اللهُ عَلَيْهِ وَسَلَّمَ
+        '\uFDFA': 'صَلَّى اللهُ عَلَيْهِ وَسَلَّمَ',
+        '\uFD46': 'صَلَّى اللهُ عَلَيْهِ وَآلِهِ وَسَلَّمَ',
+        'ﷺ': 'صَلَّى اللهُ عَلَيْهِ وَسَلَّمَ',
+        '\uF067': 'صَلَّى اللهُ عَلَيْهِ وَسَلَّمَ',
+        '\uF030': 'صَلَّى اللهُ عَلَيْهِ وَسَلَّمَ',
+        '\uF031': 'صَلَّى اللهُ عَلَيْهِ وَسَلَّمَ',
+        'PBUH': 'صَلَّى اللهُ عَلَيْهِ وَسَلَّمَ',
+        'pbuh': 'صَلَّى اللهُ عَلَيْهِ وَسَلَّمَ',
+        '(s)': 'صَلَّى اللهُ عَلَيْهِ وَسَلَّمَ',
+        '(S)': 'صَلَّى اللهُ عَلَيْهِ وَسَلَّمَ',
+        '(saw)': 'صَلَّى اللهُ عَلَيْهِ وَسَلَّمَ',
+        '(SAW)': 'صَلَّى اللهُ عَلَيْهِ وَسَلَّمَ',
+        '(PBUH)': 'صَلَّى اللهُ عَلَيْهِ وَسَلَّمَ',
+        '(pbuh)': 'صَلَّى اللهُ عَلَيْهِ وَسَلَّمَ',
+
+        # Allah ﷻ - عَزَّ وَجَلَّ
+        '\uFDFB': 'عَزَّ وَجَلَّ',
+        'ﷻ': 'عَزَّ وَجَلَّ',
+        '\uFBF2': 'عَزَّ وَجَلَّ',
+        '\uFBF1': 'عَزَّ وَجَلَّ',
+        '\uF063': 'سُبْحَانَهُ وَتَعَالَى',
+        '(swt)': 'سُبْحَانَهُ وَتَعَالَى',
+        '(SWT)': 'سُبْحَانَهُ وَتَعَالَى',
+
+        # Other prophets - عَلَيْهِ السَّلَامُ
+        '\uF064': 'عَلَيْهِ السَّلَامُ',
+        '(as)': 'عَلَيْهِ السَّلَامُ',
+        '(AS)': 'عَلَيْهِ السَّلَامُ',
+
+        # Companions - رَضِيَ اللهُ عَنْهُ
+        '\uF065': 'رَضِيَ اللهُ عَنْهُ',
+        '(ra)': 'رَضِيَ اللهُ عَنْهُ',
+        '(RA)': 'رَضِيَ اللهُ عَنْهُ',
+    }
+
+    SYMBOLS = set(HONORIFICS.keys())
+
+    @classmethod
+    def is_honorific(cls, text: str) -> bool:
+        """Check if text is an honorific symbol."""
+        return text in cls.SYMBOLS
+
+    @classmethod
+    def expand(cls, symbol: str) -> Optional[str]:
+        """Expand honorific symbol to spoken Arabic."""
+        return cls.HONORIFICS.get(symbol)
+
+    @classmethod
+    def process_text(cls, text: str) -> tuple[str, str]:
+        """
+        Process text that may contain honorifics.
+
+        Returns:
+            Tuple of (display_text, spoken_text)
+        """
+        display = text
+        spoken = text
+
+        for symbol, expansion in cls.HONORIFICS.items():
+            if symbol in text:
+                spoken = spoken.replace(symbol, expansion)
+
+        return display, spoken

versed/_data/transliterations.json

@@ -0,0 +1,45 @@
+{
+  "words": {
+    "prophet": "النبي",
+    "allah": "الله",
+    "god": "الله",
+    "quran": "القرآن",
+    "qur'an": "القرآن",
+    "hadith": "حديث",
+    "sunnah": "سنة",
+    "surah": "سورة",
+    "sura": "سورة",
+    "ayah": "آية",
+    "ayat": "آيات",
+    "hammazan": "هَمَّاز",
+    "lammazan": "لَمَّاز",
+    "ayyaban": "عَيَّاب",
+    "al-fatiha": "الفاتحة",
+    "al-baqarah": "البقرة",
+    "al-imran": "آل عمران",
+    "al-nisa": "النساء",
+    "al-ma'idah": "المائدة",
+    "al-an'am": "الأنعام",
+    "al-a'raf": "الأعراف",
+    "al-anfal": "الأنفال",
+    "al-tawbah": "التوبة",
+    "al-jathiyah": "الجاثية",
+    "al-kahf": "الكهف",
+    "al-isra": "الإسراء",
+    "al-maryam": "مريم",
+    "al-haj": "الحج",
+    "al-nur": "النور",
+    "al-furqan": "الفرقان",
+    "ya-sin": "يس",
+    "al-rahman": "الرحمن",
+    "al-mulk": "الملك",
+    "al-qalam": "القلم"
+  },
+  "prefixes": {
+    "abu": "أبو",
+    "ibn": "ابن",
+    "umm": "أم",
+    "bint": "بنت",
+    "al-": "الـ"
+  }
+}

versed/arabic.py

@@ -0,0 +1,110 @@
+"""Portable Arabic text helpers for the public engine."""
+
+from __future__ import annotations
+
+from typing import Iterable
+
+
+ARABIC_DIACRITICS = frozenset(
+    {
+        "\u064B",
+        "\u064C",
+        "\u064D",
+        "\u064E",
+        "\u064F",
+        "\u0650",
+        "\u0651",
+        "\u0652",
+        "\u0653",
+        "\u0654",
+        "\u0655",
+        "\u0656",
+        "\u0657",
+        "\u0658",
+        "\u0670",
+    }
+)
+
+ARABIC_BASE_LETTERS = frozenset(chr(codepoint) for codepoint in range(0x0621, 0x064B))
+
+
+def _iter_arabic_chars(text: str) -> Iterable[str]:
+    return (
+        char
+        for char in text
+        if "\u0600" <= char <= "\u06FF" or "\u0750" <= char <= "\u077F"
+    )
+
+
+def is_arabic(text: str) -> bool:
+    """Return True when the string contains Arabic characters."""
+    return any(True for _ in _iter_arabic_chars(text))
+
+
+def is_mostly_arabic(text: str, threshold: float = 0.5) -> bool:
+    """Return True when at least `threshold` of visible chars are Arabic."""
+    if not text:
+        return False
+
+    visible_chars = [char for char in text if not char.isspace()]
+    if not visible_chars:
+        return False
+
+    arabic_chars = [char for char in visible_chars if is_arabic(char)]
+    return (len(arabic_chars) / len(visible_chars)) >= threshold
+
+
+def strip_diacritics(text: str) -> str:
+    """Remove Arabic diacritics for fuzzy matching."""
+    return "".join(char for char in text if char not in ARABIC_DIACRITICS)
+
+
+def orphan_diacritic_rate(text: str) -> float:
+    """Measure how often diacritics appear without a valid Arabic base letter."""
+    if not text:
+        return 0.0
+
+    diacritic_count = 0
+    orphan_count = 0
+
+    for index, char in enumerate(text):
+        if char not in ARABIC_DIACRITICS:
+            continue
+
+        diacritic_count += 1
+
+        cursor = index - 1
+        while cursor >= 0 and text[cursor] in ARABIC_DIACRITICS:
+            cursor -= 1
+
+        if cursor < 0:
+            orphan_count += 1
+        elif text[cursor] not in ARABIC_BASE_LETTERS and not is_arabic(text[cursor]):
+            orphan_count += 1
+
+    return orphan_count / diacritic_count if diacritic_count else 0.0
+
+
+def detect_batch_reversal(words_raw: list) -> bool:
+    """Detect likely RTL reversal in a PyMuPDF-style `get_text(\"words\")` batch."""
+    arabic_texts = []
+    for word in words_raw:
+        text = word[4] if len(word) > 4 else ""
+        if is_arabic(text) and any(char in ARABIC_DIACRITICS for char in text):
+            arabic_texts.append(text)
+
+    if len(arabic_texts) < 3:
+        return False
+
+    orphan_rates = [orphan_diacritic_rate(text) for text in arabic_texts[:30]]
+
+    if max(orphan_rates) >= 0.4:
+        return True
+
+    words_with_orphans = sum(1 for rate in orphan_rates if rate > 0.1)
+    if words_with_orphans / len(orphan_rates) >= 0.3:
+        return True
+
+    average_rate = sum(orphan_rates) / len(orphan_rates)
+    return average_rate > 0.15
+