bytesense 1.0.0__py3-none-any.whl

bytesense/repair.py

@@ -0,0 +1,280 @@
+"""
+Mojibake repair engine.
+
+Mojibake = text that was decoded with the wrong encoding and then
+re-encoded or displayed as-is, producing garbled characters.
+
+Classic pattern: UTF-8 bytes → decoded as Latin-1 → re-encoded → "été" instead of "été"
+
+Strategy:
+  1. Re-encode the garbled string back to bytes using the assumed wrong encoding.
+  2. Try to decode those bytes with the correct encoding.
+  3. Accept the result if mess_ratio improves significantly.
+
+Supports:
+  - Single-step mojibake   (utf-8 → latin-1 → utf-8)
+  - Double-step mojibake   (utf-8 → latin-1 → utf-8 → latin-1 → utf-8)
+  - Windows-1252 variant   (utf-8 → cp1252 → utf-8)
+  - Detection-guided repair (auto-detect which transformation was applied)
+
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import List, Optional
+
+from .mess import mess_ratio
+
+# Ordered list of (re-encode-as, then-decode-as) transformation chains to try.
+# Most common patterns first.
+_REPAIR_CHAINS: list[tuple[str, str]] = [
+    ("latin_1", "utf_8"),  # UTF-8 read as Latin-1 (most common)
+    ("cp1252", "utf_8"),  # UTF-8 read as Windows-1252
+    ("latin_1", "cp1252"),  # cp1252 read as Latin-1
+    ("latin_1", "cp1251"),  # Cyrillic UTF-8 read as Latin-1
+    ("latin_1", "cp1253"),  # Greek UTF-8 read as Latin-1
+    ("latin_1", "cp1256"),  # Arabic UTF-8 read as Latin-1
+    ("latin_1", "iso8859_2"),  # Central European UTF-8 read as Latin-1
+    ("utf_8", "latin_1"),  # Latin-1 re-encoded as UTF-8 (less common)
+    ("cp1252", "latin_1"),  # Latin-1 mistaken for cp1252
+]
+
+# Maximum improvement in mess ratio to accept a repair
+_MIN_IMPROVEMENT: float = 0.10
+
+
+def _likely_utf8_mojibake(text: str) -> bool:
+    """Heuristic markers when UTF-8 was misread as Latin-1 / cp1252 (mess may stay low)."""
+    return any(m in text for m in ("Ã", "Â", "â", "Ä", "Å", "Æ", "Ç", "Ð", "Ñ", "Ø", "Ù"))
+
+
+# UTF-8 read as Latin-1 / cp1252: only these chains are used for equal-mess tie-break
+# (broader chains are still tried for strict mess improvement).
+_TIE_BREAK_CHAINS: frozenset[tuple[str, str]] = frozenset(
+    {
+        ("latin_1", "utf_8"),
+        ("cp1252", "utf_8"),
+    }
+)
+
+
+@dataclass
+class RepairResult:
+    """Result of a mojibake repair attempt."""
+
+    original: str
+    repaired: str
+    improved: bool
+    chain: Optional[tuple[str, str]]  # (re_encode, decode) or None
+    original_mess: float
+    repaired_mess: float
+    iterations: int  # Number of repair steps applied (1 = single, 2 = double)
+
+    @property
+    def improvement(self) -> float:
+        return self.original_mess - self.repaired_mess
+
+    def __str__(self) -> str:
+        if not self.improved:
+            return self.original
+        return self.repaired
+
+    def __repr__(self) -> str:
+        return (
+            f"RepairResult(improved={self.improved}, "
+            f"chain={self.chain!r}, "
+            f"improvement={self.improvement:.3f})"
+        )
+
+
+def _try_chain(text: str, re_encode: str, decode_as: str) -> Optional[str]:
+    """
+    Apply one transformation: re-encode text with `re_encode`, then decode as `decode_as`.
+    Returns repaired string or None if transformation fails or produces worse text.
+    """
+    try:
+        raw = text.encode(re_encode, errors="strict")
+        return raw.decode(decode_as, errors="strict")
+    except (UnicodeEncodeError, UnicodeDecodeError, LookupError):
+        return None
+
+
+def repair(
+    text: str,
+    max_iterations: int = 2,
+    chains: Optional[List[tuple[str, str]]] = None,
+) -> RepairResult:
+    """
+    Attempt to repair mojibake in ``text``.
+
+    Args:
+        text:           Potentially garbled string to repair.
+        max_iterations: Maximum repair cycles (1 = single chain, 2 = double).
+        chains:         Override the default transformation chain list.
+
+    Returns:
+        :class:`RepairResult` — always contains the best available result.
+        Check ``.improved`` to know if repair was applied.
+
+    Example::
+
+        from bytesense.repair import repair
+
+        garbled = "été"
+        result = repair(garbled)
+        if result.improved:
+            print(result.repaired)   # "été"
+        else:
+            print(result.original)   # unchanged
+    """
+    if not isinstance(text, str):
+        raise TypeError("text must be str")
+    if (
+        isinstance(max_iterations, bool)
+        or not isinstance(max_iterations, int)
+        or not 1 <= max_iterations <= 2
+    ):
+        raise ValueError("max_iterations must be 1 or 2")
+    if not text:
+        return RepairResult(
+            original=text,
+            repaired=text,
+            improved=False,
+            chain=None,
+            original_mess=0.0,
+            repaired_mess=0.0,
+            iterations=0,
+        )
+
+    original_mess = mess_ratio(text)
+    best_text = text
+    best_mess = original_mess
+    best_chain: Optional[tuple[str, str]] = None
+    best_iters = 0
+
+    effective_chains = chains if chains is not None else _REPAIR_CHAINS
+
+    # Single-step: collect candidates, then pick lowest mess then preferred chain order.
+    options: list[tuple[tuple[str, str], str, float, int]] = []
+    for i, (re_encode, decode_as) in enumerate(effective_chains):
+        candidate = _try_chain(text, re_encode, decode_as)
+        if candidate is None or candidate == text:
+            continue
+        candidate_mess = mess_ratio(candidate)
+        strong_improvement = (original_mess - candidate_mess) >= _MIN_IMPROVEMENT
+        tie_utf8_mojibake = (
+            _likely_utf8_mojibake(text)
+            and (re_encode, decode_as) in _TIE_BREAK_CHAINS
+            and candidate_mess <= original_mess + 1e-9
+        )
+        if strong_improvement or tie_utf8_mojibake:
+            options.append(((re_encode, decode_as), candidate, candidate_mess, i))
+
+    if options:
+        options.sort(key=lambda x: (x[2], x[3]))
+        best_chain, best_text, best_mess, _ = (
+            options[0][0],
+            options[0][1],
+            options[0][2],
+            options[0][3],
+        )
+        best_iters = 1
+
+    # Double-step repair (only if single-step improved things)
+    if (
+        max_iterations >= 2
+        and best_iters == 1
+        and (best_mess > 0.05 or _likely_utf8_mojibake(best_text))
+    ):
+        opts2: list[tuple[tuple[str, str], str, float, int]] = []
+        for i, (re_encode2, decode_as2) in enumerate(effective_chains):
+            candidate2 = _try_chain(best_text, re_encode2, decode_as2)
+            if candidate2 is None or candidate2 == best_text:
+                continue
+            candidate2_mess = mess_ratio(candidate2)
+            strong_improvement = (best_mess - candidate2_mess) >= _MIN_IMPROVEMENT
+            tie_utf8_mojibake = (
+                _likely_utf8_mojibake(best_text)
+                and (re_encode2, decode_as2) in _TIE_BREAK_CHAINS
+                and candidate2_mess <= best_mess + 1e-9
+            )
+            if strong_improvement or tie_utf8_mojibake:
+                opts2.append(((re_encode2, decode_as2), candidate2, candidate2_mess, i))
+        if opts2:
+            opts2.sort(key=lambda x: (x[2], x[3]))
+            best_chain = opts2[0][0]
+            best_text = opts2[0][1]
+            best_mess = opts2[0][2]
+            best_iters = 2
+
+    improved = best_iters > 0 and (
+        (original_mess - best_mess) >= _MIN_IMPROVEMENT
+        or (best_text != text and _likely_utf8_mojibake(text) and best_mess <= original_mess + 1e-9)
+    )
+
+    return RepairResult(
+        original=text,
+        repaired=best_text if improved else text,
+        improved=improved,
+        chain=best_chain if improved else None,
+        original_mess=original_mess,
+        repaired_mess=best_mess if improved else original_mess,
+        iterations=best_iters if improved else 0,
+    )
+
+
+def repair_bytes(
+    data: bytes,
+    encoding: Optional[str] = None,
+    *,
+    max_iterations: int = 2,
+    chains: Optional[List[tuple[str, str]]] = None,
+) -> RepairResult:
+    """
+    Repair mojibake in a byte sequence.
+
+    First decodes ``data`` with ``encoding`` (auto-detected if None),
+    then applies text-level repair.
+
+    Args:
+        data:             Byte sequence to repair.
+        encoding:         Known encoding. If None, auto-detected via ``from_bytes``.
+        max_iterations:   Forwarded to :func:`repair`.
+        chains:           Forwarded to :func:`repair`.
+
+    Returns:
+        :class:`RepairResult`
+    """
+    if encoding is None:
+        from .api import from_bytes as _fb
+
+        r = _fb(data)
+        if r.encoding is None:
+            raise UnicodeError("encoding is unknown; supply an explicit encoding")
+        enc = r.encoding
+    else:
+        enc = encoding
+    text = data.decode(enc, errors="strict")
+
+    return repair(text, max_iterations=max_iterations, chains=chains)
+
+
+def is_mojibake(text: str, threshold: float = 0.15) -> bool:
+    """
+    Quick heuristic to determine if text looks like mojibake.
+
+    Returns True if the text has high mess ratio AND attempting at least one
+    repair chain produces a significantly cleaner result.
+
+    Args:
+        text:      String to check.
+        threshold: Mess ratio above which we attempt repair and check.
+
+    Returns:
+        bool
+    """
+    if mess_ratio(text) < threshold and not _likely_utf8_mojibake(text):
+        return False
+    result = repair(text)
+    return result.improved

bytesense/scoring.py

@@ -0,0 +1,100 @@
+"""Compact character-pair evidence; optional native scoring has identical semantics."""
+
+from __future__ import annotations
+
+import gzip
+import json
+import unicodedata
+from collections import Counter
+from functools import lru_cache
+from importlib.resources import files
+from typing import Any
+
+from ._rust import is_rust_available
+
+
+@lru_cache(maxsize=1)
+def _model() -> tuple[frozenset[str], dict[str, float], Any]:
+    payload = json.loads(
+        gzip.decompress(files("bytesense.data").joinpath("language.json.gz").read_bytes())
+    )
+    letters, pairs = frozenset(payload["letters"]), payload["pairs"]
+    native = None
+    if is_rust_available():
+        from ._rust_core import NgramModel  # type: ignore[import-not-found]
+
+        native = NgramModel(payload["letters"], list(payload["pairs"].items()))
+    return letters, pairs, native
+
+
+def _prepare(text: str) -> tuple[str, float, float]:
+    if not text:
+        return "", 0.0, 0.0
+    counts = Counter(text)
+    bad = sum(n for c, n in counts.items() if not c.isprintable() and c not in "\n\r\t") / len(text)
+    symbols = sum(
+        n
+        for c, n in counts.items()
+        if c in "\\^`{|}~#"
+        or (
+            ord(c) >= 128
+            and not c.isalpha()
+            and not c.isspace()
+            and unicodedata.category(c)[0] not in "PN"
+        )
+    ) / len(text)
+    lower = text.lower()
+    table = {ord(c): c if c.isalpha() else " " for c in set(lower)}
+    return lower.translate(table), bad, symbols
+
+
+def _score_pure(
+    text: str, bad: float, symbols: float, letters: frozenset[str], pairs: dict[str, float]
+) -> float:
+    counts = Counter(text)
+    total_letters = sum(n for c, n in counts.items() if c != " ")
+    known_letters = sum(n for c, n in counts.items() if c != " " and c in letters)
+    total = 0
+    known = 0.0
+    for (a, b), n in Counter(zip(text, text[1:])).items():
+        if a == b == " ":
+            continue
+        weight = n * (1 if (a + b).isascii() else 4)
+        total += weight
+        if a + b in pairs:
+            known += weight * pairs[a + b]
+    return (
+        0.25 * known_letters / max(1, total_letters)
+        + 0.75 * known / max(1, total)
+        - bad * 5
+        - symbols * 3
+    )
+
+
+def _classify(chars: str) -> list[tuple[bool, bool, bool]]:
+    """Unicode scalar properties from this Python version; no document state."""
+    return [
+        (
+            c.isalpha(),
+            not c.isprintable() and c not in "\n\r\t",
+            c in "\\^`{|}~#"
+            or (
+                ord(c) >= 128
+                and not c.isalpha()
+                and not c.isspace()
+                and unicodedata.category(c)[0] not in "PN"
+            ),
+        )
+        for c in chars
+    ]
+
+
+def text_quality(text: str) -> tuple[float, float]:
+    """Return linguistic support and control-character ratio (neither is a probability)."""
+    letters, pairs, native = _model()
+    if native:
+        score, bad = native.quality(text, text.lower(), _classify)
+    else:
+        normalized, bad, symbols = _prepare(text)
+        score = _score_pure(normalized, bad, symbols, letters, pairs)
+    return round(score, 10), bad