arabic-fix 0.4.0__py3-none-any.whl

arabic_fix/__init__.py

@@ -0,0 +1,13 @@
+"""arabic-fix: shape, BiDi-fix, and normalize Arabic text.
+
+A unified API for the three things that go wrong when Arabic text flows
+through systems built English-first: lost letter shaping, broken BiDi
+order, and inconsistent Unicode normalization.
+"""
+
+from __future__ import annotations
+
+from .fixer import fix, fix_report, FixOptions, FixReport
+
+__all__ = ["fix", "fix_report", "FixOptions", "FixReport"]
+__version__ = "0.2.0"

arabic_fix/bidi.py

@@ -0,0 +1,86 @@
+"""Bidirectional (BiDi) reordering for mixed-direction text.
+
+Arabic is RTL. Numbers and Latin words embedded inside Arabic need
+explicit reordering or they land on the wrong side of the line.
+`python-bidi` does this correctly; we wrap it here so callers can pass a
+single string and get a display-ready one back.
+
+For HTML/CSS contexts (which is most web output), use the system's own
+BiDi engine via `direction: rtl` + `unicode-bidi: isolate` on the
+container — DO NOT pre-shape Arabic inside HTML, the browser does that
+better. This module is for terminals, log files, PDFs, AI chat where
+no rendering engine helps.
+"""
+
+from __future__ import annotations
+
+from typing import Optional
+
+try:
+    from bidi.algorithm import get_display  # type: ignore[import-untyped]
+    _HAS_BIDI = True
+    _ImportError: Exception | None = None
+except Exception as _exc:  # pragma: no cover
+    # bidi is untyped; plain `= None` is fine for mypy.
+    get_display = None
+    _HAS_BIDI = False
+    _ImportError = _exc
+
+
+# python-bidi v0.6+ uses single-char codes for base_dir
+_RTL_LTR_TO_BIDI = {"rtl": "R", "ltr": "L"}
+
+
+def reorder(
+    text: str,
+    *,
+    base_dir: str = "rtl",
+    upper_is_rtl: bool = False,
+) -> str:
+    """Apply BiDi reordering for display.
+
+    Parameters
+    ----------
+    text:
+        Input. May already be letter-shaped, but most callers pass
+        unshaped text here — shaping is a separate step.
+    base_dir:
+        'rtl' (default) or 'ltr'. Set 'ltr' if the dominant run is
+        English with an Arabic substring. Internally translated to
+        python-bidi's 'R' / 'L' codes.
+    upper_is_rtl:
+        Treat runs of uppercase Latin as RTL. Almost never what you
+        want; default False.
+
+    Returns
+    -------
+    str
+        Display-ordered string. If `python-bidi` is missing, the input
+        is returned unchanged.
+
+    Raises
+    ------
+    ValueError
+        If `base_dir` is not one of 'rtl' / 'ltr' (case-insensitive).
+        Previously this was silently passed through to `python-bidi`,
+        which would either raise an opaque error or fall back to a
+        default — masking caller bugs. Now we fail loud at the API edge.
+    """
+    if not text:
+        return text
+    if not _HAS_BIDI:
+        return text
+    bidi_dir = _RTL_LTR_TO_BIDI.get(base_dir.lower())
+    if bidi_dir is None:
+        raise ValueError(
+            f"unknown base_dir: {base_dir!r}; expected one of "
+            f"{sorted(_RTL_LTR_TO_BIDI)} (case-insensitive)"
+        )
+    return get_display(  # type: ignore
+        text,
+        base_dir=bidi_dir,
+        upper_is_rtl=upper_is_rtl,
+    )
+
+
+__all__ = ["reorder"]

arabic_fix/cli.py

@@ -0,0 +1,161 @@
+"""CLI entry point — `arabic-fix` command."""
+
+from __future__ import annotations
+
+import argparse
+import io
+import sys
+from pathlib import Path
+
+from .fixer import fix, fix_report, FixOptions
+
+
+def _build_parser() -> argparse.ArgumentParser:
+    p = argparse.ArgumentParser(
+        prog="arabic-fix",
+        description=(
+            "Fix Arabic text in files: letter-shape, BiDi-reorder, normalize. "
+            "Reads UTF-8, writes UTF-8."
+        ),
+    )
+    p.add_argument(
+        "paths",
+        nargs="*",
+        type=Path,
+        help="Files to fix. Use '-' or omit to read from stdin.",
+    )
+    p.add_argument(
+        "-o", "--output",
+        type=Path,
+        help="Output file. Only valid with a single input file. "
+             "If omitted with one input, write in place.",
+    )
+    p.add_argument("--no-shape", action="store_true", help="Skip letter shaping.")
+    p.add_argument("--no-bidi", action="store_true", help="Skip BiDi reordering.")
+    p.add_argument(
+        "--no-normalize",
+        action="store_true",
+        help="Skip Unicode normalization.",
+    )
+    p.add_argument(
+        "--normalize-form",
+        default="NFC",
+        choices=["NFC", "NFD", "NFKC", "NFKD"],
+        help="Unicode normalization form (default NFC).",
+    )
+    p.add_argument(
+        "--bidi-base-dir",
+        default="rtl",
+        choices=["rtl", "ltr"],
+        help="Base direction for BiDi reorder (default rtl).",
+    )
+    p.add_argument(
+        "--no-ligatures",
+        action="store_true",
+        help="Disable reshape ligatures (e.g. لا, ﷲ).",
+    )
+    p.add_argument(
+        "--report",
+        action="store_true",
+        help="Print a JSON report per file (what was shaped, reordered, "
+             "normalized) to stderr.",
+    )
+    p.add_argument(
+        "--check",
+        action="store_true",
+        help="Exit non-zero if any file would change. Useful in CI to "
+             "enforce Arabic text is already fixed.",
+    )
+    p.add_argument(
+        "--encoding",
+        default="utf-8",
+        help="File encoding (default utf-8).",
+    )
+    return p
+
+
+def _process(text: str, opts: FixOptions) -> str:
+    return fix(text, **opts.__dict__)
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = _build_parser()
+    args = parser.parse_args(argv)
+    opts = FixOptions(
+        shape=not args.no_shape,
+        bidi=not args.no_bidi,
+        normalize_text=not args.no_normalize,
+        normalize_form=args.normalize_form,
+        bidi_base_dir=args.bidi_base_dir,
+        ligatures=not args.no_ligatures,
+    )
+
+    if not args.paths or (len(args.paths) == 1 and str(args.paths[0]) == "-"):
+        # stdin → stdout.
+        #
+        # `sys.stdin.read()` defaults to `locale.getpreferredencoding(False)`,
+        # which on many non-English terminals is ASCII or Latin-1 and will
+        # raise `UnicodeDecodeError` on Arabic UTF-8 bytes. Force UTF-8 by
+        # wrapping the binary buffer explicitly. This matches the encoding
+        # we use for file reads (`--encoding`, default utf-8).
+        #
+        # In tests, `sys.stdin` is sometimes monkeypatched to a `StringIO`
+        # (which has no `.buffer`); fall back to that wrapped as text.
+        if hasattr(sys.stdin, "buffer"):
+            text = io.TextIOWrapper(sys.stdin.buffer, encoding="utf-8").read()
+        else:
+            text = sys.stdin.read()
+        out = _process(text, opts)
+        sys.stdout.write(out)
+        return 0
+
+    if len(args.paths) > 1 and args.output:
+        print("error: --output can only be used with a single input file.",
+              file=sys.stderr)
+        return 2
+
+    exit_code = 0
+    for path in args.paths:
+        try:
+            original = path.read_text(encoding=args.encoding)
+        except OSError as exc:
+            print(f"error: cannot read {path}: {exc}", file=sys.stderr)
+            exit_code = 1
+            continue
+
+        report = fix_report(original, **opts.__dict__)
+
+        if args.report:
+            import json
+            payload = {
+                "path": str(path),
+                "changed": report.changed,
+                "shaped": report.shaped,
+                "bidi_reordered": report.bidi_reordered,
+                "normalized": report.normalized,
+                "contained_arabic": report.contained_arabic,
+                "warnings": report.warnings,
+            }
+            print(json.dumps(payload, ensure_ascii=False), file=sys.stderr)
+
+        if args.check:
+            if report.changed:
+                print(f"would change: {path}", file=sys.stderr)
+                exit_code = 1
+            continue
+
+        if not report.changed:
+            continue
+
+        target = args.output if args.output else path
+        try:
+            target.write_text(report.output, encoding=args.encoding)
+        except OSError as exc:
+            print(f"error: cannot write {target}: {exc}", file=sys.stderr)
+            exit_code = 1
+
+    return exit_code
+
+
+if __name__ == "__main__":  # pragma: no cover
+    sys.exit(main())

arabic_fix/fixer.py

@@ -0,0 +1,163 @@
+"""The high-level `fix()` API — one call, all three fixes.
+
+Pipeline order matters: shape FIRST, then BiDi-reorder, then normalize.
+Skipping any step produces subtly-broken output in a different way:
+
+    normalize → shape  : marks get reshaped into presentation forms too
+                          late to fix; cached glyph tables may stay wrong.
+    shape → normalize → bidi : normalization AFTER shape may decompose
+                               presentation forms back to base chars.
+    bidi first         : without shaping, BiDi reorders disconnected
+                         glyphs and you can't fix it later.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+from .shaper import shape
+from .bidi import reorder
+from .normalize import normalize, contains_arabic, NormForm
+from .bidi import _RTL_LTR_TO_BIDI  # for normalizing base_dir at the API edge
+
+
+@dataclass
+class FixOptions:
+    """Tune which fixes get applied and how."""
+    shape: bool = True
+    bidi: bool = True
+    normalize_text: bool = True
+    normalize_form: str = "NFC"        # 'NFC' | 'NFD' | 'NFKC' | 'NFKD'
+    bidi_base_dir: str = "rtl"         # 'rtl' | 'ltr'
+    bidi_upper_is_rtl: bool = False
+    ligatures: bool = True             # reshape-config: presentation-form ligatures
+
+
+@dataclass
+class FixReport:
+    """Returned by `fix_report()` so callers can see what changed."""
+    input: str
+    output: str
+    shaped: bool
+    bidi_reordered: bool
+    normalized: bool
+    contained_arabic: bool
+    warnings: list[str] = field(default_factory=list)
+
+    @property
+    def changed(self) -> bool:
+        return self.input != self.output
+
+
+def _coerce_options(opts: dict[str, Any]) -> FixOptions:
+    """Build FixOptions from arbitrary user kwargs, rejecting unknown keys.
+
+    We do this rather than `FixOptions(**opts)` so a typo'd option name
+    fails loudly with a helpful message instead of silently producing
+    a dataclass missing that field.
+    """
+    known = {f for f in FixOptions.__dataclass_fields__}
+    unknown = set(opts) - known
+    if unknown:
+        raise TypeError(
+            f"unknown fix() option(s): {sorted(unknown)}; "
+            f"valid options: {sorted(known)}"
+        )
+    # Normalize bidi_base_dir to its long form ('rtl'/'ltr') if the
+    # caller passed the single-char bidi code ('R'/'L').
+    if "bidi_base_dir" in opts:
+        b = opts["bidi_base_dir"]
+        if isinstance(b, str) and b in {"R", "L"}:
+            opts["bidi_base_dir"] = "rtl" if b == "R" else "ltr"
+    # Narrow normalize_form if it was passed as a valid literal.
+    if "normalize_form" in opts:
+        f = opts["normalize_form"]
+        if f not in {"NFC", "NFD", "NFKC", "NFKD"}:
+            raise ValueError(
+                f"normalize_form must be one of NFC/NFD/NFKC/NFKD, got {f!r}"
+            )
+        opts["normalize_form"] = f
+    return FixOptions(**opts)
+
+
+def fix(text: str, **opts: Any) -> str:
+    """One-call fix: shape, BiDi-reorder, normalize.
+
+    Returns the fixed string. For diagnostics, use `fix_report()`.
+
+    Examples
+    --------
+    >>> fix("العربية")
+    '\\u0627\\u0644\\u0639\\u0631\\u0628\\u064a\\u0629'  # shaped + BiDi-fixed
+    >>> fix("Hello مرحبا 123")
+    ...  # digits 123 end up on the left in display order
+    """
+    options = _coerce_options(opts) if opts else FixOptions()
+    return _apply(text, options).output
+
+
+def fix_report(text: str, **opts: Any) -> FixReport:
+    """Same as `fix()` but returns a `FixReport` describing what happened."""
+    options = _coerce_options(opts) if opts else FixOptions()
+    return _apply(text, options)
+
+
+def _apply(text: str, options: FixOptions) -> FixReport:
+    if not text:
+        return FixReport(
+            input=text,
+            output=text,
+            shaped=False,
+            bidi_reordered=False,
+            normalized=False,
+            contained_arabic=False,
+        )
+
+    warnings: list[str] = []
+    had_arabic = contains_arabic(text)
+    out = text
+    shaped_now = False
+    bidi_now = False
+    normalized_now = False
+
+    if options.normalize_text:
+        new = normalize(out, form=options.normalize_form)
+        if new != out:
+            normalized_now = True
+            out = new
+
+    if options.shape and had_arabic:
+        try:
+            new = shape(out, ligatures=options.ligatures)
+            if new != out:
+                shaped_now = True
+                out = new
+        except Exception as exc:  # pragma: no cover
+            warnings.append(f"shaper failed: {exc}")
+
+    if options.bidi and had_arabic:
+        try:
+            new = reorder(
+                out,
+                base_dir=options.bidi_base_dir,
+                upper_is_rtl=options.bidi_upper_is_rtl,
+            )
+            if new != out:
+                bidi_now = True
+                out = new
+        except Exception as exc:  # pragma: no cover
+            warnings.append(f"bidi reorder failed: {exc}")
+
+    return FixReport(
+        input=text,
+        output=out,
+        shaped=shaped_now,
+        bidi_reordered=bidi_now,
+        normalized=normalized_now,
+        contained_arabic=had_arabic,
+        warnings=warnings,
+    )
+
+
+__all__ = ["fix", "fix_report", "FixOptions", "FixReport"]

arabic_fix/normalize.py

@@ -0,0 +1,83 @@
+"""Unicode normalization helpers.
+
+NFC vs NFD drift is a common source of "same text, different bytes" bugs
+across macOS / Windows / Linux systems handling Arabic — especially with
+tashkeel (combining marks for fatḥa, kasra, ḍamma).
+
+Default to NFC (composed) for interchange: smaller byte size, more
+predictable hashing, fully compatible with the rest of the library.
+"""
+
+from __future__ import annotations
+
+import unicodedata
+from typing import Literal
+
+
+def normalize(text: str, form: str = "NFC") -> str:
+    """Return text in the requested Unicode normalization form.
+
+    Parameters
+    ----------
+    text:
+        Any string.
+    form:
+        One of 'NFC' (default), 'NFD', 'NFKC', 'NFKD'.
+
+    Returns
+    -------
+    str
+        Normalized string. Empty input returns empty.
+    """
+    if not text:
+        return text
+    form = form.upper()
+    if form not in {"NFC", "NFD", "NFKC", "NFKD"}:
+        raise ValueError(f"unknown normalization form: {form!r}")
+    # unicodedata.normalize wants a Literal[...] for the form arg;
+    # we've validated it above, so the cast is safe.
+    return unicodedata.normalize(form, text)  # type: ignore[arg-type]
+
+
+NormForm = Literal["NFC", "NFD", "NFKC", "NFKD"]
+
+
+def contains_arabic(text: str) -> bool:
+    """Return True if `text` contains any Arabic-block codepoint."""
+    for ch in text:
+        cp = ord(ch)
+        # Arabic block U+0600–U+06FF
+        # Arabic Supplement U+0750–U+077F
+        # Arabic Extended-A U+08A0–U+08FF
+        # Arabic Presentation Forms-A U+FB50–U+FDFF
+        # Arabic Presentation Forms-B U+FE70–U+FEFF
+        if (
+            0x0600 <= cp <= 0x06FF
+            or 0x0750 <= cp <= 0x077F
+            or 0x08A0 <= cp <= 0x08FF
+            or 0xFB50 <= cp <= 0xFDFF
+            or 0xFE70 <= cp <= 0xFEFF
+        ):
+            return True
+    return False
+
+
+def has_tashkeel(text: str) -> bool:
+    """Return True if text contains any Arabic combining mark (tashkeel)."""
+    tashkeel_ranges = (
+        (0x064B, 0x065F),   # tanween, harakat, hamza-here
+        (0x0670, 0x0670),   # alef khanjariya (superscript alef)
+        (0x06D6, 0x06DC),   # Quranic marks
+        (0x06DF, 0x06E4),
+        (0x06E7, 0x06E8),
+        (0x06EA, 0x06ED),
+    )
+    for ch in text:
+        cp = ord(ch)
+        for lo, hi in tashkeel_ranges:
+            if lo <= cp <= hi:
+                return True
+    return False
+
+
+__all__ = ["normalize", "contains_arabic", "has_tashkeel"]

arabic_fix/py.typed

@@ -0,0 +1 @@
+# Marker file — the arabic_fix package supports PEP 561 type checking.

arabic_fix/shaper.py

@@ -0,0 +1,98 @@
+"""Letter shaping for Arabic script.
+
+Arabic letters change form based on position: isolated, initial, medial,
+final. Unicode's Arabic Presentation Forms-B block (U+FE70–U+FEFF) plus
+the ligature-aware reshaper `arabic-reshaper` turn a string of base
+characters into the connected glyphs the eye expects.
+
+We wrap `arabic-reshaper` with a graceful fallback that returns the
+input unchanged when the optional dependency is missing, so the rest of
+the library still works.
+
+Note: `arabic-reshaper` 3.0.0's `default_config` has
+`delete_harakat=True`. We override that to **preserve tashkeel**
+(matches `agents/eval_cases.md` Case 3 — "Tashkeel preservation").
+Stripping diacritics silently is a content-loss bug.
+"""
+
+from __future__ import annotations
+
+from typing import Optional
+
+try:  # arabic-reshaper is an optional dep at import time so the package
+      # still loads if only BiDi / normalization are needed.
+    import arabic_reshaper  # type: ignore[import-untyped]
+    from arabic_reshaper.reshaper_config import (  # type: ignore[import-untyped]
+        default_config as _default_config,
+    )
+    _HAS_RESHAPER = True
+    _ImportError: Exception | None = None
+except Exception as _exc:  # pragma: no cover
+    # mypy sees these names as Any (module is untyped), so plain `= None`
+    # works without an explicit ignore.
+    arabic_reshaper = None
+    _default_config = None
+    _HAS_RESHAPER = False
+    _ImportError = _exc
+
+
+def _build_reshaper(ligatures: bool) -> "object":
+    """Build an ArabicReshaper with tashkeel preservation + caller-controlled ligatures.
+
+    We always build our own rather than using `arabic_reshaper.reshape()`
+    so we can override `delete_harakat`. The library default strips
+    diacritics silently — we don't want that.
+
+    Returns the reshaper object typed as `object` because arabic_reshaper
+    ships no py.typed marker; we only call `.reshape(text)` on it.
+    """
+    from arabic_reshaper import ArabicReshaper  # noqa: F401  (only used at runtime)
+
+    # `default_config` is dict[str, Any] at runtime; mypy sees it as
+    # `Literal[...]` which doesn't accept dict(). Cast through Any.
+    cfg = dict(_default_config)
+    cfg["delete_harakat"] = False
+    cfg["support_ligatures"] = ligatures
+    return ArabicReshaper(configuration=cfg)
+
+
+# Cache one reshaper per (ligatures True / False). Identity cache —
+# reuse the same instance across calls so the library isn't rebuilt
+# every shape() invocation.
+_RESHAPERS: dict[bool, "object"] = {}
+
+
+def shape(text: str, *, ligatures: bool = True) -> str:
+    """Apply Arabic letter shaping.
+
+    Parameters
+    ----------
+    text:
+        Arabic (or mixed) text in its base (un-shaped) form.
+    ligatures:
+        If True (default), apply presentation-form ligatures
+        (e.g. لا, ﷲ). Turn off only for debugging or rare font issues.
+
+    Returns
+    -------
+    str
+        The shaped string. Tashkeel (diacritics) is preserved. If
+        `arabic-reshaper` is not installed, the input is returned
+        unchanged.
+    """
+    if not text:
+        return text
+    if not _HAS_RESHAPER:
+        return text  # graceful fallback: input passes through
+
+    reshaper = _RESHAPERS.get(ligatures)
+    if reshaper is None:
+        reshaper = _build_reshaper(ligatures)
+        _RESHAPERS[ligatures] = reshaper
+    # `reshaper.reshape` is typed by arabic-reshaper as Any -> Any.
+    # We promise the caller a str; assert at runtime via cast.
+    out: str = reshaper.reshape(text)  # type: ignore[attr-defined]
+    return out
+
+
+__all__ = ["shape"]