arafix 0.8.0__py3-none-any.whl

arafix/__init__.py

@@ -0,0 +1,210 @@
+"""
+arafix — recover broken Arabic text from PDFs.
+
+Graded ladder (not one hammer)::
+
+    0  diagnose()          know before you fix
+    1a fold_simple_forms() presentation forms; keep ligatures atomic
+    2  fix_order()         visual → logical, protect LTR runs
+    1b expand_ligatures()  ﻻ → لا after order is stable
+    3  build_glyph_map()   rebuild from embedded font
+    4  OCR                 last resort (not shipped)
+
+Quick start::
+
+    >>> from arafix import repair_text
+    >>> repair_text("\ufee3\ufeae\ufea3\ufe92\ufe8e").text
+    'مرحبا'
+
+    >>> from arafix import repair_blocks
+    >>> repair_blocks(["\ufee3\ufeae\ufea3\ufe92\ufe8e"]).texts[0]
+    'مرحبا'
+
+    >>> from arafix import extract_pdf           # doctest: +SKIP
+    >>> doc = extract_pdf("thesis.pdf")          # doctest: +SKIP
+
+MIT license. Primary long-form docs are in Arabic (README).
+"""
+
+from __future__ import annotations
+
+__version__ = "0.8.0"
+__license__ = "MIT"
+
+from .adapters import as_blocks, fix_any, fix_markitdown, fix_table
+from .cmap import GlyphMap, build_glyph_map, decode_glyph_name
+from .diagnose import (
+    DEFAULT_THRESHOLDS,
+    detect_mojibake,
+    detect_presentation_forms,
+    detect_pua,
+    detect_visual_order,
+    diagnose,
+)
+from .evaluate import (
+    EvalConfig,
+    EvalReport,
+    cer,
+    compare_extractors,
+    evaluate_pdf,
+    evaluate_text,
+    levenshtein,
+    levenshtein_reference,
+    wer,
+)
+from .extractors import Extractor, RawPage, get_extractor, register
+from .hygiene import (
+    count_artifacts,
+    fold_arabic_punct_confusables,
+    sanitize_extraction,
+)
+from .lamalef import (
+    LamAlefReport,
+    detect_lam_alef_transposition,
+    repair_lam_alef_transposition,
+)
+from .layout import (
+    Glyph,
+    LayoutColumn,
+    LayoutConfig,
+    LayoutLine,
+    LayoutTable,
+    PageLayout,
+    analyze_layout,
+    cluster_to_lines,
+    table_to_markdown,
+)
+from .normalize import (
+    NormalizeConfig,
+    expand_deferred_forms,
+    expand_ligatures,
+    fold_presentation_forms,
+    fold_simple_forms,
+    normalize_text,
+)
+from .order import (
+    MIRROR_PAIRS,
+    ReorderConfig,
+    fix_order,
+    grapheme_clusters,
+    reverse_visual_line,
+)
+from .pipeline import (
+    PipelineConfig,
+    extract_pdf,
+    harvest_document_lexicon,
+    repair_blocks,
+    repair_text,
+)
+from .types import (
+    BlockResult,
+    BlocksResult,
+    Defect,
+    Diagnosis,
+    DocumentResult,
+    Evidence,
+    PageResult,
+    RepairResult,
+    Stage,
+    TextBlock,
+)
+from .unicode_tables import (
+    DEFERRED_PF_TO_BASE,
+    LIGATURE_PF_TO_BASE,
+    PF_TO_BASE,
+    SIMPLE_PF_TO_BASE,
+    SPACING_MARK_PF_TO_BASE,
+    JoiningForm,
+    unicode_version,
+)
+
+__all__ = [
+    "__version__",
+    # الأنبوب
+    "repair_text",
+    "repair_blocks",
+    "extract_pdf",
+    "PipelineConfig",
+    "harvest_document_lexicon",
+    # مهايئات
+    "fix_any",
+    "fix_markitdown",
+    "fix_table",
+    "as_blocks",
+    # نظافة الاستخراج
+    "sanitize_extraction",
+    "count_artifacts",
+    "fold_arabic_punct_confusables",
+    # البنية
+    "Glyph",
+    "LayoutLine",
+    "LayoutColumn",
+    "LayoutTable",
+    "PageLayout",
+    "LayoutConfig",
+    "analyze_layout",
+    "cluster_to_lines",
+    "table_to_markdown",
+    # الدرجة ٠
+    "diagnose",
+    "detect_mojibake",
+    "detect_presentation_forms",
+    "detect_pua",
+    "detect_visual_order",
+    "DEFAULT_THRESHOLDS",
+    # الدرجة ١ (تمريرتان: مفردات ← اتجاه ← رباطات)
+    "normalize_text",
+    "fold_presentation_forms",
+    "fold_simple_forms",
+    "expand_deferred_forms",
+    "expand_ligatures",
+    "NormalizeConfig",
+    # لام-ألف
+    "detect_lam_alef_transposition",
+    "repair_lam_alef_transposition",
+    "LamAlefReport",
+    # الدرجة ٢
+    "fix_order",
+    "reverse_visual_line",
+    "grapheme_clusters",
+    "MIRROR_PAIRS",
+    "ReorderConfig",
+    # الدرجة ٣
+    "build_glyph_map",
+    "decode_glyph_name",
+    "GlyphMap",
+    # النماذج
+    "Defect",
+    "Stage",
+    "Evidence",
+    "Diagnosis",
+    "RepairResult",
+    "PageResult",
+    "DocumentResult",
+    "TextBlock",
+    "BlockResult",
+    "BlocksResult",
+    # القياس
+    "evaluate_text",
+    "evaluate_pdf",
+    "compare_extractors",
+    "cer",
+    "wer",
+    "levenshtein",
+    "levenshtein_reference",
+    "EvalConfig",
+    "EvalReport",
+    # المحرّكات
+    "Extractor",
+    "RawPage",
+    "get_extractor",
+    "register",
+    # الجداول
+    "PF_TO_BASE",
+    "SIMPLE_PF_TO_BASE",
+    "DEFERRED_PF_TO_BASE",
+    "LIGATURE_PF_TO_BASE",
+    "SPACING_MARK_PF_TO_BASE",
+    "JoiningForm",
+    "unicode_version",
+]

arafix/adapters.py

@@ -0,0 +1,83 @@
+"""
+مهايئات خفيفة — مسارٌ واحد من «نصٍّ مستخرج بأيّ أداة» إلى عربيّ سليم.
+
+    from arafix.adapters import fix_markitdown, fix_any
+
+    md = MarkItDown().convert("f.pdf")
+    print(fix_markitdown(md).text)
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from .pipeline import PipelineConfig, repair_blocks, repair_text
+from .types import BlocksResult, RepairResult, TextBlock
+
+__all__ = ["fix_any", "fix_markitdown", "fix_table", "as_blocks"]
+
+
+def fix_any(text: str, config: PipelineConfig | None = None) -> RepairResult:
+    """أيّ نصٍّ — من pdfminer أو المتصفح أو الحافظة."""
+    return repair_text(text, config)
+
+
+def fix_markitdown(
+    result: Any,
+    config: PipelineConfig | None = None,
+) -> RepairResult:
+    """
+    يقبل ``DocumentConverterResult`` من MarkItDown (أو أيّ كائن فيه
+    ``text_content`` / ``markdown``).
+    """
+    text = (
+        getattr(result, "text_content", None)
+        or getattr(result, "markdown", None)
+        or str(result)
+    )
+    return repair_text(text, config)
+
+
+def as_blocks(
+    rows: list[list[str]],
+    *,
+    id_prefix: str = "r",
+) -> list[TextBlock]:
+    """ يحوّل جدولاً (قائمة صفوف) إلى كتل ``TextBlock`` بخلايا مستقلة."""
+    blocks: list[TextBlock] = []
+    for i, row in enumerate(rows):
+        for j, cell in enumerate(row):
+            blocks.append(
+                TextBlock(
+                    text=cell or "",
+                    id=f"{id_prefix}{i}c{j}",
+                    role="cell",
+                    meta={"row": i, "col": j},
+                )
+            )
+    return blocks
+
+
+def fix_table(
+    rows: list[list[str]],
+    config: PipelineConfig | None = None,
+) -> list[list[str]]:
+    """
+    يصلح خلايا جدولٍ كلٌّ على حدة ويُرجع نفس الشكل.
+
+    >>> fix_table([["\ufee3\ufeae\ufea3\ufe92\ufe8e", "OK"]])
+    [['مرحبا', 'OK']]
+    """
+    if not rows:
+        return []
+    blocks = as_blocks(rows)
+    repaired: BlocksResult = repair_blocks(blocks, config)
+    by_id = repaired.by_id()
+    out: list[list[str]] = []
+    for i, row in enumerate(rows):
+        new_row = []
+        for j, _ in enumerate(row):
+            key = f"r{i}c{j}"
+            new_row.append(by_id[key].text if key in by_id else "")
+        out.append(new_row)
+    return out

arafix/cli.py

@@ -0,0 +1,288 @@
+"""
+واجهة سطر الأوامر.
+
+    arafix diagnose thesis.pdf
+    arafix extract  thesis.pdf -o out.txt
+    arafix eval     thesis.pdf --truth thesis.txt --compare
+    arafix text     "ﺎﺒﺣﺮﻣ"
+    arafix fonts    thesis.pdf
+
+فلسفة الأمر `diagnose` أنه **لا يكتب شيئاً**. اقرأ تقريره أولاً، ثم
+قرّر. الأداة التي تعالج قبل أن تُريك ما وجدت أداةٌ لا تُؤتمن.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+
+from . import __version__
+from .diagnose import diagnose
+from .pipeline import PipelineConfig, extract_pdf, repair_text
+from .unicode_tables import unicode_version
+
+
+def _cmd_diagnose(args: argparse.Namespace) -> int:
+    from .extractors import get_extractor
+
+    ex = get_extractor(args.extractor)
+    report = []
+    for raw in ex.pages(args.path):
+        dg = diagnose(raw.text)
+        report.append(
+            {
+                "page": raw.number,
+                "chars": dg.char_count,
+                "arabic_ratio": round(dg.arabic_ratio, 3),
+                "defects": [d.value for d in dg.defects],
+                "confidence": dg.confidence,
+                "defect_confidence": {
+                    k.value: v for k, v in dg.defect_confidence.items()
+                },
+                "fonts": raw.fonts,
+                "evidence": [
+                    {"name": e.name, "value": round(e.value, 3), "detail": e.detail}
+                    for e in dg.evidence
+                ],
+            }
+        )
+        if args.pages and raw.number >= args.pages:
+            break
+
+    if args.json:
+        print(json.dumps(report, ensure_ascii=False, indent=2))
+        return 0
+
+    for r in report:
+        print(f"── صفحة {r['page']} " + "─" * 40)
+        conf = r["defect_confidence"]
+        detail = "، ".join(f"{d} ({conf.get(d, 0):.2f})" for d in r["defects"])
+        print(f"   العلل    : {detail}")
+        print(f"   الثقة    : {r['confidence']}  (أضعف حلقة)")
+        print(f"   الحروف   : {r['chars']}  (عربية {r['arabic_ratio']:.0%})")
+        if r["fonts"]:
+            print(f"   الخطوط   : {', '.join(r['fonts'][:4])}")
+        if args.verbose:
+            for e in r["evidence"]:
+                print(f"     · {e['name']:22s} {e['value']:+.3f}  {e['detail']}")
+    return 0
+
+
+def _cmd_extract(args: argparse.Namespace) -> int:
+    from .layout import LayoutConfig
+
+    lay_cfg = LayoutConfig(reading_order=args.reading_order)
+    cfg = PipelineConfig(
+        extractor=args.extractor,
+        force_reorder=args.force_reorder,
+        layout=args.layout,
+        layout_config=lay_cfg,
+    )
+    doc = extract_pdf(args.path, cfg)
+
+    out = doc.text
+    if args.output:
+        with open(args.output, "w", encoding="utf-8") as fh:
+            fh.write(out)
+        print(f"كُتب {len(out)} حرفاً في {args.output}", file=sys.stderr)
+    else:
+        print(out)
+
+    print(f"الثقة الدنيا عبر {len(doc.pages)} صفحة: {doc.confidence}", file=sys.stderr)
+    if args.verbose:
+        print(
+            f"بنية: أعمدة≤{doc.metadata.get('max_columns', 1)} "
+            f"جداول={doc.metadata.get('table_count', 0)} "
+            f"layout={doc.metadata.get('layout')}",
+            file=sys.stderr,
+        )
+        for p in doc.pages:
+            if p.n_columns > 1 or p.tables:
+                print(
+                    f"  صفحة {p.page_number}: {p.n_columns} عمود، "
+                    f"{len(p.tables)} جدول",
+                    file=sys.stderr,
+                )
+        if args.tables and doc.all_tables:
+            print("\n── جداول ──", file=sys.stderr)
+            for i, grid in enumerate(doc.all_tables):
+                print(f"جدول {i + 1}: {len(grid)}×{len(grid[0]) if grid else 0}", file=sys.stderr)
+                for row in grid:
+                    print("  | " + " | ".join(row) + " |")
+    if doc.confidence < 0.5:
+        print("تحذير: ثقة منخفضة — راجع `arafix diagnose -v`", file=sys.stderr)
+        return 2
+    return 0
+
+
+def _cmd_text(args: argparse.Namespace) -> int:
+    src = args.text if args.text else sys.stdin.read()
+    r = repair_text(src)
+    print(r.text)
+    if args.verbose:
+        print(f"\n─ العلل: {r.diagnosis.summary()}", file=sys.stderr)
+        print(f"─ المراحل: {[s.value for s in r.stages_applied]}", file=sys.stderr)
+        print(f"─ الثقة: {r.confidence}", file=sys.stderr)
+        for n in r.notes:
+            print(f"  · {n}", file=sys.stderr)
+    return 0
+
+
+def _cmd_blocks(args: argparse.Namespace) -> int:
+    """أصلح سطوراً/خلايا من stdin (سطر = كتلة) — للجداول والأنابيب."""
+    from .pipeline import repair_blocks
+    from .types import TextBlock
+
+    lines = [ln.rstrip("\n\r") for ln in sys.stdin]
+    if args.skip_empty:
+        lines = [ln for ln in lines if ln.strip()]
+    blocks = [TextBlock(text=ln, id=f"L{i}", role="line") for i, ln in enumerate(lines)]
+    out = repair_blocks(blocks)
+    for b in out.blocks:
+        print(b.text)
+    if args.verbose:
+        print(
+            f"─ {len(out.blocks)} كتلة · ثقة دنيا {out.confidence}",
+            file=sys.stderr,
+        )
+        n_changed = sum(1 for b in out.blocks if b.repair.changed)
+        print(f"─ تغيّر منها: {n_changed}", file=sys.stderr)
+    return 0
+
+
+def _cmd_eval(args: argparse.Namespace) -> int:
+    from .evaluate import EvalConfig, compare_extractors, evaluate_pdf
+
+    cfg = EvalConfig(
+        ignore_diacritics=args.ignore_diacritics,
+        ignore_punctuation=args.ignore_punctuation,
+    )
+    reports = (
+        compare_extractors(args.path, args.truth, cfg)
+        if args.compare
+        else [evaluate_pdf(args.path, args.truth, args.extractor, cfg)]
+    )
+
+    print("─" * 68)
+    for r in reports:
+        print(" ", r)
+    print("─" * 68)
+
+    best = reports[0]
+    if args.verbose and best.worst_lines:
+        print("\nأسوأ السطور في", best.label, ":")
+        for i, ref, hyp in best.worst_lines:
+            print(f"  سطر {i}")
+            print(f"    المرجع : {ref[:70]!r}")
+            print(f"    الناتج : {hyp[:70]!r}")
+
+    if len(reports) > 1:
+        gap = reports[-1].cer.rate - best.cer.rate
+        print(f"\nأفضل مسار: {best.label} — يسبق أسوأهم بـ {gap:.2%} في CER")
+    return 0 if best.cer.rate < 0.05 else 3
+
+
+def _cmd_fonts(args: argparse.Namespace) -> int:
+    from .cmap import build_glyph_map
+    from .extractors import get_extractor
+
+    ex = get_extractor(args.extractor)
+    fonts = ex.font_bytes(args.path)
+    if not fonts:
+        print("لا خطوط مضمَّنة — الدرجة ٣ غير ممكنة على هذا الملف.")
+        return 1
+    for name, data in fonts.items():
+        try:
+            gm = build_glyph_map(data, name)
+            print(f"{name:40s} تغطية {gm.coverage:.0%}  ثقة {gm.confidence}  ({gm.source})")
+            for note in gm.notes:
+                print(f"    ! {note}")
+        except Exception as exc:
+            print(f"{name:40s} تعذّر التحليل: {exc}")
+    return 0
+
+
+def build_parser() -> argparse.ArgumentParser:
+    p = argparse.ArgumentParser(
+        prog="arafix", description="استرجاع النص العربي من ملفات PDF المعطوبة"
+    )
+    p.add_argument(
+        "--version",
+        action="version",
+        version=f"arafix {__version__} · unicode {unicode_version()}",
+    )
+    p.add_argument("-e", "--extractor", default="auto", help="محرّك القراءة (auto|pymupdf)")
+    sub = p.add_subparsers(dest="cmd", required=True)
+
+    d = sub.add_parser("diagnose", help="شخّص ولا تكتب شيئاً")
+    d.add_argument("path")
+    d.add_argument("-v", "--verbose", action="store_true", help="اعرض الشواهد")
+    d.add_argument("--json", action="store_true")
+    d.add_argument("-n", "--pages", type=int, default=0, help="حدّ الصفحات")
+    d.set_defaults(func=_cmd_diagnose)
+
+    x = sub.add_parser("extract", help="استخرج وأصلح")
+    x.add_argument("path")
+    x.add_argument("-o", "--output")
+    x.add_argument("--force-reorder", action="store_true", help="اعكس بلا شاهد")
+    x.add_argument(
+        "--layout",
+        choices=["auto", "linear", "columns", "full"],
+        default="auto",
+        help="تحليل البنية: auto|linear|columns|full",
+    )
+    x.add_argument(
+        "--reading-order",
+        choices=["rtl", "ltr"],
+        default="rtl",
+        help="ترتيب قراءة الأعمدة (افتراضي rtl)",
+    )
+    x.add_argument("-v", "--verbose", action="store_true", help="اعرض ملخص البنية")
+    x.add_argument("--tables", action="store_true", help="اطبع الجداول المستخرجة")
+    x.set_defaults(func=_cmd_extract)
+
+    t = sub.add_parser("text", help="أصلح نصاً مباشراً أو من stdin")
+    t.add_argument("text", nargs="?")
+    t.add_argument("-v", "--verbose", action="store_true")
+    t.set_defaults(func=_cmd_text)
+
+    b = sub.add_parser(
+        "blocks",
+        help="أصلح كتلًا مستقلة من stdin (سطر=كتلة) — جداول وأنابيب",
+    )
+    b.add_argument("-v", "--verbose", action="store_true")
+    b.add_argument(
+        "--skip-empty",
+        action="store_true",
+        help="تجاهل الأسطر الفارغة",
+    )
+    b.set_defaults(func=_cmd_blocks)
+
+    v = sub.add_parser("eval", help="قِس مقابل حقيقةٍ مرجعية (CER/WER)")
+    v.add_argument("path")
+    v.add_argument("--truth", required=True, help="ملفٌ نصّيّ فيه النصّ الصحيح")
+    v.add_argument("--compare", action="store_true", help="قِس كل المسارات ورتّبها")
+    v.add_argument("--ignore-diacritics", action="store_true")
+    v.add_argument("--ignore-punctuation", action="store_true")
+    v.add_argument("-v", "--verbose", action="store_true", help="اسرد أسوأ السطور")
+    v.set_defaults(func=_cmd_eval)
+
+    f = sub.add_parser("fonts", help="افحص الخطوط المضمَّنة (الدرجة ٣)")
+    f.add_argument("path")
+    f.set_defaults(func=_cmd_fonts)
+
+    return p
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = build_parser().parse_args(argv)
+    try:
+        return args.func(args)
+    except (RuntimeError, KeyError, FileNotFoundError) as exc:
+        print(f"خطأ: {exc}", file=sys.stderr)
+        return 1
+
+
+if __name__ == "__main__":  # pragma: no cover
+    sys.exit(main())