arabicpdf 0.1.0__py3-none-any.whl

arabicpdf/__init__.py

@@ -0,0 +1,25 @@
+"""ArabicPDF — Convert any Arabic PDF to text. Lightweight. Local. Accurate."""
+
+from __future__ import annotations
+
+from arabicpdf.config import (
+    Config,
+    ConversionResult,
+    Mode,
+    PageResult,
+    PageType,
+)
+from arabicpdf.core import convert, to_text
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "to_text",
+    "convert",
+    "Config",
+    "Mode",
+    "PageType",
+    "PageResult",
+    "ConversionResult",
+    "__version__",
+]

arabicpdf/arabic_utils.py

@@ -0,0 +1,326 @@
+"""
+Arabic text utilities for ArabicPDF.
+
+Provides Unicode-aware helpers for detecting, analysing, and tokenising
+Arabic text. All functions are pure-Python with no external dependencies.
+
+Key capabilities
+─────────────────
+• Unicode range checks (is_arabic_char, is_arabic_word, is_valid_arabic)
+• Text analysis (arabic_char_ratio, count_arabic_words, has_diacritics, char_entropy)
+• Dot-confusion data (DOT_GROUPS, CONFUSION_MAP, variant sets)
+• Levenshtein edit-distance (no external deps)
+• Arabic-aware tokenisation
+"""
+
+from __future__ import annotations
+
+import math
+import re
+from collections import Counter
+from typing import Final
+
+# ──────────────────────────────────────────────────────────────────────
+# 1. Unicode range checks
+# ──────────────────────────────────────────────────────────────────────
+
+#: Compiled pattern matching the main Arabic Unicode blocks:
+#:   U+0600–U+06FF  Arabic
+#:   U+0750–U+077F  Arabic Supplement
+#:   U+FB50–U+FDFF  Arabic Presentation Forms-A
+#:   U+FE70–U+FEFF  Arabic Presentation Forms-B
+ARABIC_PATTERN: Final[re.Pattern[str]] = re.compile(
+    r"[\u0600-\u06FF\u0750-\u077F\uFB50-\uFDFF\uFE70-\uFEFF]"
+)
+
+
+def is_arabic_char(char: str) -> bool:
+    """Check whether a single character falls within Arabic Unicode ranges.
+
+    Parameters
+    ----------
+    char:
+        A single character string.
+
+    Returns
+    -------
+    bool
+        ``True`` if the character is in one of the Arabic Unicode blocks.
+
+    Raises
+    ------
+    ValueError
+        If *char* is not exactly one character long.
+    """
+    if len(char) != 1:
+        raise ValueError(f"Expected a single character, got {len(char)} characters")
+    return ARABIC_PATTERN.match(char) is not None
+
+
+def is_arabic_word(word: str) -> bool:
+    """Check whether a word is *predominantly* Arabic.
+
+    A word is considered Arabic when more than half of its non-whitespace
+    characters belong to Arabic Unicode ranges.
+
+    Parameters
+    ----------
+    word:
+        The word to test.
+
+    Returns
+    -------
+    bool
+        ``True`` if the word is predominantly Arabic.
+    """
+    if not word:
+        return False
+    chars = [ch for ch in word if not ch.isspace()]
+    if not chars:
+        return False
+    arabic_count = sum(1 for ch in chars if ARABIC_PATTERN.match(ch))
+    return arabic_count > len(chars) / 2
+
+
+def is_valid_arabic(text: str, threshold: float = 0.6) -> bool:
+    """Decide whether *text* is mostly valid Arabic.
+
+    Computes the ratio of Arabic characters to total non-space characters
+    and checks that it meets or exceeds *threshold*.
+
+    Parameters
+    ----------
+    text:
+        The text to evaluate.
+    threshold:
+        Minimum ratio of Arabic characters (default ``0.6``).
+
+    Returns
+    -------
+    bool
+        ``True`` if the Arabic-character ratio ≥ *threshold*.
+    """
+    return arabic_char_ratio(text) >= threshold
+
+
+# ──────────────────────────────────────────────────────────────────────
+# 2. Arabic text analysis
+# ──────────────────────────────────────────────────────────────────────
+
+def arabic_char_ratio(text: str) -> float:
+    """Return the ratio of Arabic characters to total alphabetic letters.
+
+    Punctuation (e.g. leader dots in table of contents) and digits
+    do not penalise the ratio.
+
+    Parameters
+    ----------
+    text:
+        Input text.
+
+    Returns
+    -------
+    float
+        A value in [0.0, 1.0].  Returns ``0.0`` for empty / whitespace-only
+        text.
+    """
+    letters = [ch for ch in text if ch.isalpha()]
+    if letters:
+        arabic_letters = sum(1 for ch in letters if ARABIC_PATTERN.match(ch))
+        return arabic_letters / len(letters)
+
+    non_space = [ch for ch in text if not ch.isspace()]
+    if not non_space:
+        return 0.0
+    arabic_count = sum(1 for ch in non_space if ARABIC_PATTERN.match(ch))
+    return arabic_count / len(non_space)
+
+
+def count_arabic_words(text: str) -> int:
+    """Count the number of words in *text* that are predominantly Arabic.
+
+    Words are split on whitespace via :func:`str.split`.
+
+    Parameters
+    ----------
+    text:
+        Input text.
+
+    Returns
+    -------
+    int
+        Number of Arabic words.
+    """
+    return sum(1 for word in text.split() if is_arabic_word(word))
+
+
+#: Arabic diacritical marks (tashkeel), U+064B – U+0652.
+_DIACRITICS_PATTERN: Final[re.Pattern[str]] = re.compile(r"[\u064B-\u0652]")
+
+
+def has_diacritics(text: str) -> bool:
+    """Check whether *text* contains any Arabic diacritics (tashkeel).
+
+    The diacritics checked are U+064B (fathatan) through U+0652 (sukun).
+
+    Parameters
+    ----------
+    text:
+        Input text.
+
+    Returns
+    -------
+    bool
+        ``True`` if at least one diacritical mark is present.
+    """
+    return _DIACRITICS_PATTERN.search(text) is not None
+
+
+def char_entropy(text: str) -> float:
+    """Compute the Shannon entropy of the character distribution in *text*.
+
+    Garbled / incorrectly-decoded text tends to have abnormally high or
+    low entropy compared to natural Arabic.
+
+    Parameters
+    ----------
+    text:
+        Input text (whitespace is included in the distribution).
+
+    Returns
+    -------
+    float
+        Entropy in bits.  Returns ``0.0`` for empty text.
+    """
+    if not text:
+        return 0.0
+    counts = Counter(text)
+    total = len(text)
+    entropy = 0.0
+    for count in counts.values():
+        prob = count / total
+        if prob > 0:
+            entropy -= prob * math.log2(prob)
+    return entropy
+
+
+# ──────────────────────────────────────────────────────────────────────
+# 3. Arabic dot-confusion data
+# ──────────────────────────────────────────────────────────────────────
+
+#: Groups of Arabic letters that share the same base shape and differ
+#: only in the number / placement of dots.
+DOT_GROUPS: Final[list[str]] = [
+    "بتثنی",   # ba / ta / tha / nun / ya (Farsi)
+    "جحخ",      # jim / ḥa / kha
+    "دذ",       # dal / dhal
+    "رز",       # ra / zay
+    "سش",       # sin / shin
+    "صض",       # ṣad / ḍad
+    "طظ",       # ṭa / ẓa
+    "عغ",       # ʿain / ghain
+    "فق",       # fa / qaf
+]
+
+#: Map each Arabic letter to the set of letters it can be confused with
+#: (same base shape, different dots).  Built from :data:`DOT_GROUPS`.
+CONFUSION_MAP: Final[dict[str, set[str]]] = {}
+for _group in DOT_GROUPS:
+    for _char in _group:
+        CONFUSION_MAP[_char] = set(_group) - {_char}
+
+#: Common alef variants that may need normalisation.
+ALEF_VARIANTS: Final[str] = "اأإآٱ"
+
+#: Ta marbuta and Ha — often confused in OCR output.
+TA_MARBUTA_HA: Final[str] = "ةه"
+
+#: Alef maqsura and Ya — often confused in OCR output.
+ALEF_MAQSURA_YA: Final[str] = "ىي"
+
+
+# ──────────────────────────────────────────────────────────────────────
+# 4. Levenshtein distance
+# ──────────────────────────────────────────────────────────────────────
+
+def levenshtein(s1: str, s2: str) -> int:
+    """Compute the Levenshtein (edit) distance between two strings.
+
+    Uses the classic dynamic-programming algorithm with *O(min(m, n))*
+    space (single-row optimisation).  No external dependencies.
+
+    Parameters
+    ----------
+    s1:
+        First string.
+    s2:
+        Second string.
+
+    Returns
+    -------
+    int
+        Minimum number of single-character edits (insertions, deletions,
+        or substitutions) needed to transform *s1* into *s2*.
+    """
+    # Ensure s1 is the shorter string for space optimisation.
+    if len(s1) > len(s2):
+        s1, s2 = s2, s1
+
+    n = len(s1)
+    m = len(s2)
+
+    # Previous row of distances.
+    previous: list[int] = list(range(n + 1))
+
+    for j in range(1, m + 1):
+        current: list[int] = [j] + [0] * n
+        for i in range(1, n + 1):
+            cost = 0 if s1[i - 1] == s2[j - 1] else 1
+            current[i] = min(
+                current[i - 1] + 1,       # insertion
+                previous[i] + 1,           # deletion
+                previous[i - 1] + cost,    # substitution
+            )
+        previous = current
+
+    return previous[n]
+
+
+# ──────────────────────────────────────────────────────────────────────
+# 5. Text tokenisation
+# ──────────────────────────────────────────────────────────────────────
+
+#: Arabic-specific punctuation characters that should act as token
+#: boundaries in addition to whitespace.
+_ARABIC_PUNCTUATION: Final[str] = (
+    "\u060C"   # Arabic comma ،
+    "\u061B"   # Arabic semicolon ؛
+    "\u061F"   # Arabic question mark ؟
+    "\u06D4"   # Arabic full stop ۔
+    "\u0640"   # tatweel (kashida) ـ
+)
+
+#: Pattern that splits on whitespace and Arabic / Latin punctuation,
+#: keeping only non-empty tokens.
+_TOKENIZE_PATTERN: Final[re.Pattern[str]] = re.compile(
+    r"[" + re.escape(_ARABIC_PUNCTUATION) + r"\s\.\,\;\:\!\?\"\'\(\)\[\]\{\}<>«»/\\]+"
+)
+
+
+def tokenize_arabic(text: str) -> list[str]:
+    """Split *text* into word tokens, handling Arabic punctuation.
+
+    Splits on whitespace as well as common Arabic and Latin punctuation.
+    Empty tokens are discarded.
+
+    Parameters
+    ----------
+    text:
+        Input text (may contain mixed Arabic / Latin content).
+
+    Returns
+    -------
+    list[str]
+        Non-empty word tokens, in their original order.
+    """
+    return [tok for tok in _TOKENIZE_PATTERN.split(text) if tok]

arabicpdf/cli.py

@@ -0,0 +1,168 @@
+"""Command-line interface for ArabicPDF."""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from pathlib import Path
+
+from arabicpdf import __version__, convert, to_text, Config, Mode
+from arabicpdf.detector import classify_document
+
+
+def _parse_pages(pages_str: str | None) -> list[int] | None:
+    """Parse a page range string like '1,3,5-7' into a sorted list of ints."""
+    if not pages_str:
+        return None
+    pages: set[int] = set()
+    for part in pages_str.split(","):
+        part = part.strip()
+        if "-" in part:
+            start_s, end_s = part.split("-", 1)
+            start, end = int(start_s.strip()), int(end_s.strip())
+            pages.update(range(start, end + 1))
+        else:
+            pages.add(int(part))
+    return sorted(pages)
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(
+        prog="arabicpdf",
+        description="Convert any Arabic PDF to clean text locally with high accuracy.",
+    )
+    parser.add_argument(
+        "-v", "--version", action="version", version=f"%(prog)s {__version__}"
+    )
+
+    subparsers = parser.add_subparsers(dest="command", help="Available subcommands")
+
+    # Command: convert
+    convert_parser = subparsers.add_parser(
+        "convert", help="Extract text from an Arabic PDF"
+    )
+    convert_parser.add_argument(
+        "pdf_path", type=str, help="Path to input Arabic PDF"
+    )
+    convert_parser.add_argument(
+        "-o", "--output", type=str, default=None, help="Output text file path (default: stdout)"
+    )
+    convert_parser.add_argument(
+        "-m",
+        "--mode",
+        choices=["fast", "balanced", "max_accuracy"],
+        default="balanced",
+        help="Extraction mode (default: balanced)",
+    )
+    convert_parser.add_argument(
+        "-p", "--pages", type=str, default=None, help="Page numbers or ranges (e.g. '1,3,5-8')"
+    )
+    convert_parser.add_argument(
+        "--dpi", type=int, default=300, help="Rendering DPI for OCR (default: 300)"
+    )
+    convert_parser.add_argument(
+        "--strip-diacritics",
+        action="store_true",
+        help="Remove Arabic vowels/harakat from output",
+    )
+    convert_parser.add_argument(
+        "--normalize-alef",
+        action="store_true",
+        help="Normalize all alef forms (أ/إ/آ) to bare alef (ا)",
+    )
+    convert_parser.add_argument(
+        "--no-spell-check",
+        action="store_true",
+        help="Disable dictionary-based spell and dot correction",
+    )
+    convert_parser.add_argument(
+        "--details",
+        action="store_true",
+        help="Print detailed page-level extraction metadata",
+    )
+
+    # Command: inspect / info
+    inspect_parser = subparsers.add_parser(
+        "inspect", help="Inspect PDF page types (digital vs scanned vs corrupted)"
+    )
+    inspect_parser.add_argument(
+        "pdf_path", type=str, help="Path to input Arabic PDF"
+    )
+
+    args = parser.parse_args(argv)
+
+    if not args.command:
+        # Default behavior: if a single positional argument is passed without subcommand
+        if len(sys.argv) > 1 and not sys.argv[1].startswith("-"):
+            args.command = "convert"
+            args.pdf_path = sys.argv[1]
+            args.output = None
+            args.mode = "balanced"
+            args.pages = None
+            args.dpi = 300
+            args.strip_diacritics = False
+            args.normalize_alef = False
+            args.no_spell_check = False
+            args.details = False
+        else:
+            parser.print_help()
+            return 1
+
+    if args.command == "inspect":
+        pdf_file = Path(args.pdf_path)
+        if not pdf_file.is_file():
+            sys.stderr.write(f"Error: File not found: {args.pdf_path}\n")
+            return 1
+        print(f"Inspecting '{args.pdf_path}'...\n")
+        classifications = classify_document(str(pdf_file))
+        for c in classifications:
+            print(
+                f"Page {c.page_num:3d}: Type={c.page_type.value:18s} "
+                f"Confidence={c.confidence*100:5.1f}%  HasImages={str(c.has_images):5s}"
+            )
+        return 0
+
+    if args.command == "convert":
+        pdf_file = Path(args.pdf_path)
+        if not pdf_file.is_file():
+            sys.stderr.write(f"Error: File not found: {args.pdf_path}\n")
+            return 1
+
+        pages_list = _parse_pages(args.pages)
+        cfg = Config(
+            mode=Mode(args.mode),
+            pages=pages_list,
+            preserve_diacritics=not args.strip_diacritics,
+            normalize_alef=args.normalize_alef,
+            fix_spelling=not args.no_spell_check,
+            fix_dot_confusion=not args.no_spell_check,
+            render_dpi=args.dpi,
+        )
+
+        result = convert(str(pdf_file), config=cfg)
+
+        if args.details:
+            sys.stderr.write(f"Processed {result.page_count} page(s). Document confidence: {result.confidence*100:.1f}%\n")
+            for p in result.pages:
+                flags_msg = f" | Flags: {', '.join(p.flags)}" if p.flags else ""
+                sys.stderr.write(
+                    f" - Page {p.number}: Method={p.method} Conf={p.confidence*100:.1f}%{flags_msg}\n"
+                )
+
+        if args.output:
+            out_path = Path(args.output)
+            out_path.parent.mkdir(parents=True, exist_ok=True)
+            out_path.write_text(result.text, encoding="utf-8")
+            sys.stderr.write(f"Saved extracted text to: {args.output}\n")
+        else:
+            sys.stdout.write(result.text)
+            if not result.text.endswith("\n"):
+                sys.stdout.write("\n")
+
+        return 0
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

arabicpdf/config.py

@@ -0,0 +1,141 @@
+"""Configuration for ArabicPDF pipeline.
+
+Uses Pydantic when installed, with graceful fallback to standard library
+dataclasses for ultra-lightweight environments.
+"""
+
+from __future__ import annotations
+
+from enum import Enum
+from typing import Optional, Any
+
+try:
+    from pydantic import BaseModel, Field
+    HAS_PYDANTIC = True
+except ImportError:
+    HAS_PYDANTIC = False
+    from dataclasses import dataclass, field
+
+
+class Mode(str, Enum):
+    """Processing mode — trades speed for accuracy."""
+
+    FAST = "fast"  # Qalam only (digital) or single OCR (scanned)
+    BALANCED = "balanced"  # Dual OCR + consensus + correction
+    MAX_ACCURACY = "max_accuracy"  # All available engines + full correction
+
+
+class PageType(str, Enum):
+    """Classification of a PDF page."""
+
+    DIGITAL_CLEAN = "digital_clean"  # Valid extractable Arabic text
+    DIGITAL_CORRUPTED = "digital_corrupted"  # Text layer exists but garbled
+    SCANNED = "scanned"  # Image-only, no text layer
+    MIXED = "mixed"  # Some regions digital, some scanned
+
+
+if HAS_PYDANTIC:
+    class Config(BaseModel):
+        """Pipeline configuration."""
+
+        mode: Mode = Field(default=Mode.BALANCED, description="Processing mode")
+        pages: Optional[list[int]] = Field(
+            default=None, description="Pages to process (1-indexed). None = all."
+        )
+        preserve_diacritics: bool = Field(
+            default=True, description="Preserve tashkeel/harakat"
+        )
+        normalize_alef: bool = Field(
+            default=False, description="Normalize أ/إ/آ → ا"
+        )
+        fix_spelling: bool = Field(
+            default=True, description="Apply Hunspell spell correction"
+        )
+        fix_dot_confusion: bool = Field(
+            default=True, description="Apply Arabic dot-confusion resolver"
+        )
+        render_dpi: int = Field(
+            default=300,
+            ge=150,
+            le=1200,
+            description="DPI for rendering PDF pages to images",
+        )
+        arabic_threshold: float = Field(
+            default=0.6,
+            ge=0.0,
+            le=1.0,
+            description="Minimum ratio of Arabic chars to consider text valid",
+        )
+        correction_max_distance: int = Field(
+            default=2,
+            ge=1,
+            le=5,
+            description="Max Levenshtein distance for spell corrections",
+        )
+
+    class PageResult(BaseModel):
+        """Result for a single page."""
+
+        number: int
+        text: str
+        method: str  # "direct", "ocr", "ocr+corrected"
+        confidence: float  # 0.0 to 1.0
+        page_type: PageType
+        flags: list[str] = Field(default_factory=list)
+
+    class ConversionResult(BaseModel):
+        """Full conversion result."""
+
+        text: str
+        pages: list[PageResult]
+        confidence: float  # Average across pages
+
+        @property
+        def page_count(self) -> int:
+            return len(self.pages)
+
+        @property
+        def total_pages(self) -> int:
+            return len(self.pages)
+
+else:
+    @dataclass
+    class Config:
+        """Pipeline configuration (dataclass fallback)."""
+
+        mode: Mode = Mode.BALANCED
+        pages: Optional[list[int]] = None
+        preserve_diacritics: bool = True
+        normalize_alef: bool = False
+        fix_spelling: bool = True
+        fix_dot_confusion: bool = True
+        render_dpi: int = 300
+        arabic_threshold: float = 0.6
+        correction_max_distance: int = 2
+
+    @dataclass
+    class PageResult:
+        """Result for a single page (dataclass fallback)."""
+
+        number: int
+        text: str
+        method: str
+        confidence: float
+        page_type: PageType
+        flags: list[str] = field(default_factory=list)
+
+    @dataclass
+    class ConversionResult:
+        """Full conversion result (dataclass fallback)."""
+
+        text: str
+        pages: list[PageResult]
+        confidence: float
+
+        @property
+        def page_count(self) -> int:
+            return len(self.pages)
+
+        @property
+        def total_pages(self) -> int:
+            return len(self.pages)