bytesense 1.0.0__py3-none-any.whl

bytesense/candidate.py

@@ -0,0 +1,199 @@
+"""
+Candidate encoding selector.
+
+Reduces ~99 possible encodings to a short list before decode+mess,
+using only byte-level evidence.
+"""
+
+from __future__ import annotations
+
+import array
+from typing import List, Optional
+
+from .constant import ALL_ENCODINGS, BOM_MARKERS, SIMILAR_ENCODINGS
+from .fingerprint import (
+    byte_histogram,
+    cp1252_zone_ratio,
+    detect_null_pattern,
+    high_byte_ratio,
+    null_byte_ratio,
+    shortlist_encodings,
+    utf8_continuation_score,
+)
+from .heuristics import reorder_candidates
+
+
+def _looks_like_iso2022_bytes(data: bytes) -> bool:
+    """7-bit ISO-2022 uses ESC $ / ESC ( sequences — not plain ASCII text."""
+    if b"\x1b" not in data:
+        return False
+    return b"\x1b\x24" in data or b"\x1b\x28" in data or b"\x1b\x29" in data or b"\x1b\x2e" in data
+
+
+class CandidateSelector:
+    """
+    Select the most likely encoding candidates for a byte sequence
+    without performing any full decode.
+    """
+
+    def __init__(self, data: bytes) -> None:
+        self.data = data
+        self.length = len(data)
+        self._hist: Optional[array.array] = None
+        self._ascii_only: Optional[bool] = None
+        self._utf8_valid: Optional[bool] = None
+
+    @property
+    def hist(self) -> array.array:
+        if self._hist is None:
+            self._hist = byte_histogram(self.data)
+        return self._hist
+
+    def bom_encoding(self) -> Optional[str]:
+        """Return the encoding if a BOM/SIG prefix is detected, else None."""
+        for encoding, bom in sorted(BOM_MARKERS.items(), key=lambda x: -len(x[1])):
+            if self.data.startswith(bom):
+                return encoding
+        return None
+
+    def is_ascii_only(self) -> bool:
+        if self._ascii_only is None:
+            n = self.length
+            if n == 0:
+                self._ascii_only = True
+            else:
+                self._ascii_only = self.data.isascii() and b"\x00" not in self.data
+        return self._ascii_only
+
+    def is_utf8_valid(self) -> bool:
+        if self._utf8_valid is None:
+            try:
+                self.data.decode("utf_8")
+                self._utf8_valid = True
+            except UnicodeDecodeError:
+                self._utf8_valid = False
+        return self._utf8_valid
+
+    def get_candidates(self) -> List[str]:
+        """
+        Return ordered list of encoding candidates, most likely first.
+
+        Decision tree:
+          1. BOM  → single winner
+          2. ASCII-only → [ascii, utf_8]
+          3. Valid UTF-8 → [utf_8, …]
+          4. Null byte pattern → UTF-16/32 variants
+          5. Byte fingerprint cosine similarity → top 12
+        """
+        bom = self.bom_encoding()
+        if bom:
+            return [bom]
+
+        if self.is_ascii_only() and not _looks_like_iso2022_bytes(self.data):
+            return ["ascii", "utf_8"]
+
+        # UTF-16/32 shape must come before valid-UTF-8: ASCII in UTF-16 is often valid UTF-8 (NUL bytes).
+        null_pat = detect_null_pattern(self.data)
+        if null_pat:
+            if "32" in null_pat:
+                return [null_pat, "utf_32", "utf_32_be", "utf_32_le"]
+            return [null_pat, "utf_16", "utf_16_be", "utf_16_le"]
+
+        if self.is_utf8_valid() and not _looks_like_iso2022_bytes(self.data):
+            candidates = ["utf_8"]
+            # Check for CJK 3-byte sequences (0xE2-0xE4 range is CJK in UTF-8)
+            cjk_signal = sum(self.hist[i] for i in range(0xE2, 0xE5))
+            if cjk_signal > self.length * 0.05:
+                candidates += ["gb18030", "big5", "shift_jis"]
+            return list(dict.fromkeys(candidates))
+
+        hbr = high_byte_ratio(self.hist, self.length)
+        nbr = null_byte_ratio(self.hist, self.length)
+        c1252 = cp1252_zone_ratio(self.hist, self.length)
+        u8s = utf8_continuation_score(self.data[:4096])
+
+        # Heavy null presence → UTF-16/32 only
+        if nbr > 0.15:
+            return ["utf_16", "utf_16_le", "utf_16_be", "utf_32", "utf_32_le", "utf_32_be"]
+
+        # Strong UTF-8 continuation but invalid → likely truncated or damaged UTF-8
+        exclude: set[str] = set()
+        if u8s > 0.8:
+            exclude.update(["shift_jis", "euc_kr", "johab", "cp949"])
+
+        # Fingerprint-based shortlist
+        scored = shortlist_encodings(self.hist, self.length, top_n=20)
+        candidates = [enc for enc, _ in scored if enc not in exclude]
+
+        if _looks_like_iso2022_bytes(self.data):
+            for enc in ("iso2022_jp", "iso2022_jp_2004", "euc_jp", "shift_jis", "cp932"):
+                if enc in exclude:
+                    continue
+                if enc not in candidates:
+                    candidates.insert(0, enc)
+
+        # SBCS / common legacy encodings must be reachable even when fingerprints rank MBCS first.
+        # Include big5/cp949/euc_kr — short cosine list often omits them for small files.
+        if hbr > 0.02:
+            preferred: List[str] = []
+            for enc in (
+                "latin_1",
+                "cp1252",
+                "cp1250",
+                "cp1254",
+                "cp1256",
+                "cp1255",
+                "cp1257",
+                "cp1258",
+                "iso8859_8",
+                "cp1251",
+                "koi8_r",
+                "koi8_u",
+                "cp866",
+                "cp1253",
+                "mac_cyrillic",
+                "tis_620",
+                "iso8859_11",
+                "big5",
+                "big5hkscs",
+                "gb2312",
+                "gbk",
+                "gb18030",
+                "shift_jis",
+                "euc_jp",
+                "iso2022_jp",
+                "cp949",
+                "euc_kr",
+                "johab",
+                "iso8859_7",
+            ):
+                if enc in exclude:
+                    continue
+                if enc not in preferred:
+                    preferred.append(enc)
+            candidates = preferred + [c for c in candidates if c not in preferred]
+
+        # Ensure cp1252 is always tried for high-byte European content
+        if hbr > 0.05 and c1252 >= 0.001 and "cp1252" not in candidates:
+            candidates.insert(0, "cp1252")
+
+        # Deduplicate while preserving order
+        seen: set[str] = set()
+        final: List[str] = []
+        for enc in candidates:
+            if enc not in seen:
+                seen.add(enc)
+                final.append(enc)
+
+        final = reorder_candidates(self.data, final)
+
+        return final[:40] if final else ALL_ENCODINGS[:40]
+
+    def exclude_similar_to_failed(
+        self,
+        failed: str,
+        remaining: List[str],
+    ) -> List[str]:
+        """Remove encodings too similar to `failed` from `remaining`."""
+        similar = set(SIMILAR_ENCODINGS.get(failed, []))
+        return [enc for enc in remaining if enc not in similar]

bytesense/cli.py

@@ -0,0 +1,73 @@
+"""Command-line detection with machine-readable output and meaningful exit codes."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+from typing import List, Optional
+
+from .api import from_fp, from_path
+from .version import __version__
+
+
+def main(argv: Optional[List[str]] = None) -> int:
+    parser = argparse.ArgumentParser(
+        prog="bytesense", description="Detect encodings and validate the complete input."
+    )
+    parser.add_argument(
+        "files", nargs="+", metavar="FILE", help="File(s) to analyse; - reads binary stdin"
+    )
+    parser.add_argument("-v", "--verbose", action="store_true", help="Include the explanation")
+    parser.add_argument("-m", "--minimal", action="store_true", help="Print encoding name only")
+    parser.add_argument("--language", action="store_true", help="Include a language estimate")
+    parser.add_argument(
+        "--sample-size",
+        type=int,
+        default=4096,
+        help="Linguistic sample budget (default: 4096 bytes)",
+    )
+    parser.add_argument(
+        "--min-confidence", type=float, default=0.0, help="Abstain below this evidence score (0–1)"
+    )
+    parser.add_argument("--version", action="version", version=f"bytesense {__version__}")
+    args = parser.parse_args(argv)
+    if args.sample_size < 64 or not 0 <= args.min_confidence <= 1:
+        parser.error("sample-size must be at least 64; min-confidence must be between 0 and 1")
+    results = []
+    exit_code = 0
+    for filepath in args.files:
+        options = dict(
+            include_language=args.language,
+            sample_size=args.sample_size,
+            min_confidence=args.min_confidence,
+        )
+        try:
+            result = (
+                from_fp(sys.stdin.buffer, **options)
+                if filepath == "-"
+                else from_path(filepath, **options)
+            )
+        except OSError as exc:
+            print(f"Error: {exc}", file=sys.stderr)
+            exit_code = 1
+            continue
+        if result.encoding is None:
+            exit_code = 1
+        if args.minimal:
+            print(result.encoding or "unknown")
+            continue
+        d = result.to_dict()
+        d["path"] = "-" if filepath == "-" else str(Path(filepath).resolve())
+        if not args.verbose:
+            d.pop("why", None)
+        results.append(d)
+    if not args.minimal:
+        output = results[0] if len(results) == 1 else results
+        print(json.dumps(output, indent=2, ensure_ascii=False))
+    return exit_code
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

bytesense/coherence.py

@@ -0,0 +1,68 @@
+"""Language evidence from shared character statistics, without retaining user text."""
+
+from __future__ import annotations
+
+from collections import Counter
+from typing import Optional
+
+from .constant import CHAR_FREQUENCIES
+
+# Only static language metadata is retained globally. Document text is never cached.
+_RANKS = {
+    language: {c: i for i, c in enumerate(chars)} for language, chars in CHAR_FREQUENCIES.items()
+}
+
+
+def _score(
+    counter: Counter[str],
+    language: str,
+    ordered: Optional[list[tuple[str, int]]] = None,
+    total: Optional[int] = None,
+) -> float:
+    ranks = _RANKS.get(language)
+    if not ranks or not counter:
+        return 0.0
+    score = weight_sum = 0.0
+    total = sum(counter.values()) if total is None else total
+    covered = 0
+    for position, (char, count) in enumerate(
+        (counter.most_common() if ordered is None else ordered)[: len(ranks) + 5]
+    ):
+        weight = 1.0 / (1 + position)
+        weight_sum += weight
+        if char in ranks:
+            score += weight / (1 + abs(position - ranks[char]) * 0.08)
+            covered += count
+    # Unknown letters are evidence too. The old denominator ignored them and
+    # reported perfect Arabic coherence for mostly Latin mojibake.
+    return (score / weight_sum) * (covered / total) ** 0.5 if weight_sum else 0.0
+
+
+def _letters(text: str) -> Counter[str]:
+    counts = Counter(text.lower())
+    return Counter({char: n for char, n in counts.items() if char.isalpha()})
+
+
+def coherence_score(text: str, language: str) -> float:
+    """Heuristic language support in [0, 1], not a probability."""
+    return _score(_letters(text), language)
+
+
+def detect_language(
+    text: str,
+    candidates: Optional[list[str]] = None,
+    threshold: float = 0.1,
+) -> list[tuple[str, float]]:
+    """Rank languages using one character count for the entire candidate set."""
+    counts = _letters(text)
+    ordered = counts.most_common()
+    total = sum(counts.values())
+    result = [
+        (language, _score(counts, language, ordered, total))
+        for language in (_RANKS if candidates is None else candidates)
+    ]
+    return sorted(
+        ((language, score) for language, score in result if score >= threshold),
+        key=lambda item: item[1],
+        reverse=True,
+    )