versed-pdf 1.1.0__tar.gz

versed_pdf-1.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Sama Team
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

versed_pdf-1.1.0/PKG-INFO

@@ -0,0 +1,73 @@
+Metadata-Version: 2.4
+Name: versed-pdf
+Version: 1.1.0
+Summary: Semantic PDF-to-Markdown engine for Arabic and bilingual texts with local routing and repair
+Author: Versed Team
+License: MIT
+Keywords: arabic,pdf,markdown,text-extraction,quran,ocr,nlp
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Text Processing :: Linguistic
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Provides-Extra: pdf
+Requires-Dist: pymupdf>=1.24.0; extra == "pdf"
+Provides-Extra: ocr
+Requires-Dist: pytesseract>=0.3.10; extra == "ocr"
+Requires-Dist: Pillow>=10.0.0; extra == "ocr"
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0; extra == "dev"
+Dynamic: license-file
+
+# versed
+
+Local PDF-to-Markdown tooling for Arabic and bilingual texts.
+
+It repairs broken extraction, decodes QCF Quran fonts, classifies pages, and renders semantic Markdown from local PDFs.
+
+## Install
+
+```bash
+pip install versed-pdf
+pip install versed-pdf[pdf]
+pip install versed-pdf[pdf,ocr]
+```
+
+## Quick start
+
+```python
+from versed import extract_document
+
+result = extract_document("book.pdf", title="Book")
+print(result.markdown)
+```
+
+## CLI
+
+```bash
+versed repair-text "taf߬l"
+versed detect book.pdf
+versed classify book.pdf
+versed extract book.pdf -o book.md
+```
+
+## Public modules
+
+- `versed.repair`: Sabon mojibake repair helpers
+- `versed.qcf`: QCF Quran font decoding
+- `versed.classify`: local page classification and backend selection
+- `versed.routing`: cost-aware routing heuristics
+- `versed.layout`: aligned words to semantic blocks
+- `versed.markdown`: semantic blocks to Markdown/plain text
+- `versed.extract`: end-to-end local extraction
+
+## License
+
+MIT

versed_pdf-1.1.0/README.md

@@ -0,0 +1,45 @@
+# versed
+
+Local PDF-to-Markdown tooling for Arabic and bilingual texts.
+
+It repairs broken extraction, decodes QCF Quran fonts, classifies pages, and renders semantic Markdown from local PDFs.
+
+## Install
+
+```bash
+pip install versed-pdf
+pip install versed-pdf[pdf]
+pip install versed-pdf[pdf,ocr]
+```
+
+## Quick start
+
+```python
+from versed import extract_document
+
+result = extract_document("book.pdf", title="Book")
+print(result.markdown)
+```
+
+## CLI
+
+```bash
+versed repair-text "taf߬l"
+versed detect book.pdf
+versed classify book.pdf
+versed extract book.pdf -o book.md
+```
+
+## Public modules
+
+- `versed.repair`: Sabon mojibake repair helpers
+- `versed.qcf`: QCF Quran font decoding
+- `versed.classify`: local page classification and backend selection
+- `versed.routing`: cost-aware routing heuristics
+- `versed.layout`: aligned words to semantic blocks
+- `versed.markdown`: semantic blocks to Markdown/plain text
+- `versed.extract`: end-to-end local extraction
+
+## License
+
+MIT

versed_pdf-1.1.0/pyproject.toml

@@ -0,0 +1,47 @@
+[project]
+name = "versed-pdf"
+version = "1.1.0"
+description = "Semantic PDF-to-Markdown engine for Arabic and bilingual texts with local routing and repair"
+readme = "README.md"
+license = {text = "MIT"}
+requires-python = ">=3.9"
+authors = [
+    {name = "Versed Team"}
+]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Topic :: Text Processing :: Linguistic",
+]
+keywords = ["arabic", "pdf", "markdown", "text-extraction", "quran", "ocr", "nlp"]
+
+dependencies = []  # Zero required deps! Pure Python for string repair.
+
+[project.optional-dependencies]
+pdf = ["pymupdf>=1.24.0"]
+ocr = ["pytesseract>=0.3.10", "Pillow>=10.0.0"]
+dev = ["pytest>=7.0"]
+
+[project.scripts]
+versed = "versed.cli:main"
+
+[build-system]
+requires = ["setuptools>=65.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.setuptools.package-data]
+versed = ["_data/*.json", "_data/*.db", "py.typed"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+python_files = ["test_*.py"]
+addopts = "-v --tb=short"

versed_pdf-1.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

versed_pdf-1.1.0/src/versed/__init__.py

@@ -0,0 +1,144 @@
+"""versed — local PDF-to-Markdown tooling for Arabic and bilingual texts."""
+
+__version__ = "1.1.0"
+
+from .arabic import (
+    detect_batch_reversal,
+    is_arabic,
+    is_mostly_arabic,
+    orphan_diacritic_rate,
+    strip_diacritics,
+)
+from .classify import (
+    BackendConfig,
+    PageProbe,
+    PageType,
+    classify_and_select,
+    classify_page,
+    select_backend,
+)
+from .detect import (
+    KNOWN_MOJIBAKE_CHARS,
+    MojibakeReport,
+    detect_mojibake,
+    detect_mojibake_in_pdf,
+)
+from .extract import ExtractResult, extract_document
+from .health import summarize_text_health
+from .honorifics import (
+    HONORIFIC_SYMBOLS,
+    NormalizedWord,
+    annotate_transliterations,
+    decode_honorific,
+    expand_honorifics,
+    find_transliteration,
+    get_spoken_text,
+    has_leading_honorific,
+    normalize_text,
+    normalize_words,
+)
+from .layout import (
+    document_from_aligned_words,
+    document_from_markdown,
+    document_from_structured,
+)
+from .markdown import (
+    EnhancedMarkdownResult,
+    build_enhanced_markdown,
+    compute_aligned_words_checksum,
+    compute_cache_key,
+)
+from .qcf import (
+    QCFDecoder,
+    QCFVerse,
+    QCFWord,
+    build_qcf_mapping_from_quran_data,
+    detect_qcf_regions,
+    extract_qcf_page_number,
+    is_qcf_glyph,
+    is_qcf_text,
+)
+from .repair import (
+    SABON_CHAR_REPAIR,
+    SABON_FONT_PREFIXES,
+    extract_repairable_font_spans,
+    find_font_for_word,
+    is_repairable_font,
+    repair_text,
+    repair_text_for_font,
+    repair_words_with_font_info,
+)
+from .routing import (
+    EnrichmentDecision,
+    PageObservations,
+    TaskNeeds,
+    observe_from_extraction,
+    observe_page,
+    route_enrichment,
+)
+from .types import AlignedWord, BlockType, Document, TextBlock, WordBox
+
+__all__ = [
+    "AlignedWord",
+    "BackendConfig",
+    "BlockType",
+    "Document",
+    "EnhancedMarkdownResult",
+    "EnrichmentDecision",
+    "ExtractResult",
+    "HONORIFIC_SYMBOLS",
+    "KNOWN_MOJIBAKE_CHARS",
+    "MojibakeReport",
+    "NormalizedWord",
+    "PageObservations",
+    "PageProbe",
+    "PageType",
+    "QCFDecoder",
+    "QCFVerse",
+    "QCFWord",
+    "SABON_CHAR_REPAIR",
+    "SABON_FONT_PREFIXES",
+    "TaskNeeds",
+    "TextBlock",
+    "WordBox",
+    "annotate_transliterations",
+    "build_enhanced_markdown",
+    "build_qcf_mapping_from_quran_data",
+    "classify_and_select",
+    "classify_page",
+    "compute_aligned_words_checksum",
+    "compute_cache_key",
+    "decode_honorific",
+    "detect_batch_reversal",
+    "detect_mojibake",
+    "detect_mojibake_in_pdf",
+    "detect_qcf_regions",
+    "document_from_aligned_words",
+    "document_from_markdown",
+    "document_from_structured",
+    "expand_honorifics",
+    "extract_document",
+    "extract_qcf_page_number",
+    "extract_repairable_font_spans",
+    "find_font_for_word",
+    "find_transliteration",
+    "get_spoken_text",
+    "has_leading_honorific",
+    "is_arabic",
+    "is_mostly_arabic",
+    "is_qcf_glyph",
+    "is_qcf_text",
+    "is_repairable_font",
+    "normalize_text",
+    "normalize_words",
+    "observe_from_extraction",
+    "observe_page",
+    "orphan_diacritic_rate",
+    "repair_text",
+    "repair_text_for_font",
+    "repair_words_with_font_info",
+    "route_enrichment",
+    "select_backend",
+    "strip_diacritics",
+    "summarize_text_health",
+]

versed_pdf-1.1.0/src/versed/_arabic.py

@@ -0,0 +1,213 @@
+"""
+Arabic text utilities — normalization, detection, similarity.
+
+Internal module used by honorifics.py and detect.py.
+Not part of the public API.
+"""
+
+import re
+import unicodedata
+from typing import Dict, Optional
+
+
+class TextUtils:
+    """Text processing utilities."""
+
+    # Arabic character ranges
+    ARABIC_PATTERN = re.compile(r'[\u0600-\u06FF\u0750-\u077F\uFB50-\uFDFF\uFE70-\uFEFF]')
+
+    # Punctuation and diacritics to strip
+    PUNCTUATION_PATTERN = re.compile(
+        r"[\'\"`´′″‹›«»\u2018\u2019\u201C\u201D\u02BC\u2032\u2033.,;:!?()[\]{}،؛؟–—-]"
+    )
+    DIACRITICS_PATTERN = re.compile(r'[\u064B-\u065F\u0670]')
+
+    # Alef variants (أإآٱ) → bare alef (ا)
+    ALEF_VARIANTS = re.compile(r'[أإآٱ]')
+    # Zero-width and directional markers to drop
+    ZERO_WIDTH_TRANSLATION = str.maketrans({
+        "\u200B": "",  # zero width space
+        "\u200C": "",  # zero width non-joiner
+        "\u200D": "",  # zero width joiner
+        "\u2060": "",  # word joiner
+        "\uFEFF": "",  # zero width no-break space
+        "\u200E": "",  # LRM
+        "\u200F": "",  # RLM
+    })
+    # Common Latin ligatures (extra safety; NFKC should also handle these)
+    LIGATURE_TRANSLATION = str.maketrans({
+        "\uFB00": "ff",
+        "\uFB01": "fi",
+        "\uFB02": "fl",
+        "\uFB03": "ffi",
+        "\uFB04": "ffl",
+        "\uFB05": "st",
+        "\uFB06": "st",
+    })
+
+    @classmethod
+    def normalize_arabic(cls, text: str) -> str:
+        """
+        Normalize Arabic text for comparison/matching.
+
+        - Removes diacritics (tashkeel)
+        - Normalizes alef variants to bare alef
+        - Normalizes teh marbuta to heh
+        - Normalizes alef maksura to yeh
+        - Removes tatweel (kashida)
+        """
+        if not text:
+            return ""
+
+        # Remove tashkeel (Arabic diacritics)
+        text = cls.DIACRITICS_PATTERN.sub('', text)
+
+        # Normalize alef variants to bare alef
+        text = cls.ALEF_VARIANTS.sub('ا', text)
+
+        # Normalize teh marbuta to heh
+        text = text.replace('ة', 'ه')
+
+        # Normalize alef maksura to yeh
+        text = text.replace('ى', 'ي')
+
+        # Remove tatweel (kashida)
+        text = text.replace('\u0640', '')
+
+        return text.strip()
+
+    @classmethod
+    def normalize(cls, text: str) -> str:
+        """
+        Normalize text for matching.
+
+        - Lowercases (for non-Arabic)
+        - Strips whitespace
+        - Removes punctuation
+        - Applies full Arabic normalization (diacritics, alef variants, etc.)
+        """
+        if not text:
+            return ""
+        text = unicodedata.normalize("NFKC", text).strip()
+        text = text.translate(cls.ZERO_WIDTH_TRANSLATION)
+        text = text.translate(cls.LIGATURE_TRANSLATION)
+        text = cls.PUNCTUATION_PATTERN.sub("", text)
+
+        # Apply full Arabic normalization
+        text = cls.normalize_arabic(text)
+
+        # Lowercase for non-Arabic matching (safe after Arabic normalization)
+        text = text.lower()
+
+        return text.strip()
+
+    @classmethod
+    def is_arabic(cls, text: str) -> bool:
+        """Check if text contains Arabic characters."""
+        return bool(cls.ARABIC_PATTERN.search(text)) if text else False
+
+    @classmethod
+    def levenshtein_similarity(cls, s1: str, s2: str) -> float:
+        """
+        Calculate similarity score between two strings.
+
+        Returns:
+            Similarity score 0.0-1.0 (1.0 = identical)
+        """
+        s1, s2 = cls.normalize(s1), cls.normalize(s2)
+        if not s1 or not s2:
+            return 0.0
+        if s1 == s2:
+            return 1.0
+
+        len1, len2 = len(s1), len(s2)
+        if len1 > len2:
+            s1, s2 = s2, s1
+            len1, len2 = len2, len1
+
+        current_row = list(range(len1 + 1))
+        for i in range(1, len2 + 1):
+            previous_row, current_row = current_row, [i] + [0] * len1
+            for j in range(1, len1 + 1):
+                add = previous_row[j] + 1
+                delete = current_row[j - 1] + 1
+                change = previous_row[j - 1] + (0 if s1[j - 1] == s2[i - 1] else 1)
+                current_row[j] = min(add, delete, change)
+
+        distance = current_row[len1]
+        return 1.0 - (distance / max(len1, len2))
+
+
+class HonorificsHandler:
+    """
+    Handler for Islamic honorific symbols.
+
+    Maps display symbols (ﷺ, ﷻ, etc.) to spoken Arabic phrases.
+    """
+
+    # Honorific expansions WITH HARAKAT for proper TTS pronunciation
+    HONORIFICS: Dict[str, str] = {
+        # Prophet Muhammad ﷺ - صَلَّى اللهُ عَلَيْهِ وَسَلَّمَ
+        '\uFDFA': 'صَلَّى اللهُ عَلَيْهِ وَسَلَّمَ',
+        '\uFD46': 'صَلَّى اللهُ عَلَيْهِ وَآلِهِ وَسَلَّمَ',
+        'ﷺ': 'صَلَّى اللهُ عَلَيْهِ وَسَلَّمَ',
+        '\uF067': 'صَلَّى اللهُ عَلَيْهِ وَسَلَّمَ',
+        '\uF030': 'صَلَّى اللهُ عَلَيْهِ وَسَلَّمَ',
+        '\uF031': 'صَلَّى اللهُ عَلَيْهِ وَسَلَّمَ',
+        'PBUH': 'صَلَّى اللهُ عَلَيْهِ وَسَلَّمَ',
+        'pbuh': 'صَلَّى اللهُ عَلَيْهِ وَسَلَّمَ',
+        '(s)': 'صَلَّى اللهُ عَلَيْهِ وَسَلَّمَ',
+        '(S)': 'صَلَّى اللهُ عَلَيْهِ وَسَلَّمَ',
+        '(saw)': 'صَلَّى اللهُ عَلَيْهِ وَسَلَّمَ',
+        '(SAW)': 'صَلَّى اللهُ عَلَيْهِ وَسَلَّمَ',
+        '(PBUH)': 'صَلَّى اللهُ عَلَيْهِ وَسَلَّمَ',
+        '(pbuh)': 'صَلَّى اللهُ عَلَيْهِ وَسَلَّمَ',
+
+        # Allah ﷻ - عَزَّ وَجَلَّ
+        '\uFDFB': 'عَزَّ وَجَلَّ',
+        'ﷻ': 'عَزَّ وَجَلَّ',
+        '\uFBF2': 'عَزَّ وَجَلَّ',
+        '\uFBF1': 'عَزَّ وَجَلَّ',
+        '\uF063': 'سُبْحَانَهُ وَتَعَالَى',
+        '(swt)': 'سُبْحَانَهُ وَتَعَالَى',
+        '(SWT)': 'سُبْحَانَهُ وَتَعَالَى',
+
+        # Other prophets - عَلَيْهِ السَّلَامُ
+        '\uF064': 'عَلَيْهِ السَّلَامُ',
+        '(as)': 'عَلَيْهِ السَّلَامُ',
+        '(AS)': 'عَلَيْهِ السَّلَامُ',
+
+        # Companions - رَضِيَ اللهُ عَنْهُ
+        '\uF065': 'رَضِيَ اللهُ عَنْهُ',
+        '(ra)': 'رَضِيَ اللهُ عَنْهُ',
+        '(RA)': 'رَضِيَ اللهُ عَنْهُ',
+    }
+
+    SYMBOLS = set(HONORIFICS.keys())
+
+    @classmethod
+    def is_honorific(cls, text: str) -> bool:
+        """Check if text is an honorific symbol."""
+        return text in cls.SYMBOLS
+
+    @classmethod
+    def expand(cls, symbol: str) -> Optional[str]:
+        """Expand honorific symbol to spoken Arabic."""
+        return cls.HONORIFICS.get(symbol)
+
+    @classmethod
+    def process_text(cls, text: str) -> tuple[str, str]:
+        """
+        Process text that may contain honorifics.
+
+        Returns:
+            Tuple of (display_text, spoken_text)
+        """
+        display = text
+        spoken = text
+
+        for symbol, expansion in cls.HONORIFICS.items():
+            if symbol in text:
+                spoken = spoken.replace(symbol, expansion)
+
+        return display, spoken

versed_pdf-1.1.0/src/versed/_data/transliterations.json

@@ -0,0 +1,45 @@
+{
+  "words": {
+    "prophet": "النبي",
+    "allah": "الله",
+    "god": "الله",
+    "quran": "القرآن",
+    "qur'an": "القرآن",
+    "hadith": "حديث",
+    "sunnah": "سنة",
+    "surah": "سورة",
+    "sura": "سورة",
+    "ayah": "آية",
+    "ayat": "آيات",
+    "hammazan": "هَمَّاز",
+    "lammazan": "لَمَّاز",
+    "ayyaban": "عَيَّاب",
+    "al-fatiha": "الفاتحة",
+    "al-baqarah": "البقرة",
+    "al-imran": "آل عمران",
+    "al-nisa": "النساء",
+    "al-ma'idah": "المائدة",
+    "al-an'am": "الأنعام",
+    "al-a'raf": "الأعراف",
+    "al-anfal": "الأنفال",
+    "al-tawbah": "التوبة",
+    "al-jathiyah": "الجاثية",
+    "al-kahf": "الكهف",
+    "al-isra": "الإسراء",
+    "al-maryam": "مريم",
+    "al-haj": "الحج",
+    "al-nur": "النور",
+    "al-furqan": "الفرقان",
+    "ya-sin": "يس",
+    "al-rahman": "الرحمن",
+    "al-mulk": "الملك",
+    "al-qalam": "القلم"
+  },
+  "prefixes": {
+    "abu": "أبو",
+    "ibn": "ابن",
+    "umm": "أم",
+    "bint": "بنت",
+    "al-": "الـ"
+  }
+}