bytesense 1.0.0__py3-none-any.whl

bytesense/fingerprint.py

@@ -0,0 +1,246 @@
+"""
+Byte-distribution fingerprinting engine.
+
+The central insight: every encoding has a characteristic byte-frequency
+"signature". By computing the cosine similarity between the observed
+byte histogram and pre-computed encoding fingerprints, we can shortlist
+likely encodings in O(n) time without any decoding.
+"""
+
+from __future__ import annotations
+
+import array
+from collections import Counter
+from functools import lru_cache
+from typing import List, Tuple
+
+# ---------------------------------------------------------------------------
+# Core histogram (may be replaced by Rust at module load time — see _rust.py)
+# ---------------------------------------------------------------------------
+
+
+def _byte_histogram_pure(data: bytes) -> array.array:
+    """Pure-Python byte histogram (also used when the Rust extension is absent)."""
+    hist: array.array = array.array("Q", [0] * 256)
+    for value, count in Counter(data).items():
+        hist[value] = count
+    return hist
+
+
+def histogram_to_ratios(hist: array.array, total: int) -> List[float]:
+    """Convert raw counts to frequency ratios."""
+    if total == 0:
+        return [0.0] * 256
+    inv = 1.0 / total
+    return [c * inv for c in hist]
+
+
+def high_byte_ratio(hist: array.array, total: int) -> float:
+    """Fraction of bytes with value >= 0x80."""
+    if total == 0:
+        return 0.0
+    return sum(hist[0x80:]) / total
+
+
+def null_byte_ratio(hist: array.array, total: int) -> float:
+    """Fraction of 0x00 bytes."""
+    if total == 0:
+        return 0.0
+    return hist[0] / total
+
+
+def cp1252_zone_ratio(hist: array.array, total: int) -> float:
+    """
+    Fraction of bytes in 0x80–0x9F.
+    Non-zero → almost certainly cp1252 family, NOT latin_1
+    (latin_1 treats 0x80-0x9F as C1 control codes; cp1252 maps them to printable chars).
+    """
+    if total == 0:
+        return 0.0
+    return sum(hist[0x80:0xA0]) / total
+
+
+def _utf8_continuation_score_pure(data: bytes) -> float:
+    """
+    Pure-Python UTF-8 continuation scoring (also used when Rust is absent).
+    Score how well `data` fits UTF-8 multibyte sequence structure.
+    Returns 0.0 (no evidence) to 1.0 (strong UTF-8 multibyte pattern).
+    Works even when the data is not fully valid UTF-8.
+    """
+    if not data:
+        return 0.0
+
+    valid = 0
+    invalid = 0
+    i = 0
+    n = len(data)
+
+    while i < n:
+        b = data[i]
+        if b < 0x80:
+            i += 1
+            continue
+        elif 0xC2 <= b <= 0xDF:
+            seq_len = 2
+        elif 0xE0 <= b <= 0xEF:
+            seq_len = 3
+        elif 0xF0 <= b <= 0xF4:
+            seq_len = 4
+        else:
+            invalid += 1
+            i += 1
+            continue
+
+        if i + seq_len > n:
+            invalid += 1
+            i += 1
+            continue
+
+        ok = all(0x80 <= data[i + j] <= 0xBF for j in range(1, seq_len))
+        if ok:
+            valid += 1
+            i += seq_len
+        else:
+            invalid += 1
+            i += 1
+
+    total = valid + invalid
+    return valid / total if total > 0 else 0.0
+
+
+def detect_null_pattern(data: bytes) -> str | None:
+    """
+    Detect UTF-16/32 from null-byte distribution pattern.
+    Returns encoding name or None.
+    """
+    if len(data) < 8:
+        return None
+
+    sample = data[:512]
+    # Distinguish byte lanes, including non-Latin UTF-16 and supplementary UTF-32.
+    lanes = [sample[i::4] for i in range(4)]
+    zero = [lane.count(0) / len(lane) for lane in lanes]
+    if zero[0] > 0.8 and zero[1] > 0.8 and zero[3] < 0.5:
+        return "utf_32_be"
+    if zero[2] > 0.8 and zero[3] > 0.8 and zero[0] < 0.5:
+        return "utf_32_le"
+    even, odd = sample[0::2], sample[1::2]
+    ze, zo = even.count(0) / len(even), odd.count(0) / len(odd)
+    if ze >= 0.08 and zo < ze * 0.1:
+        return "utf_16_be"
+    if zo >= 0.08 and ze < zo * 0.1:
+        return "utf_16_le"
+    return None
+
+
+def _cosine_similarity(a: List[float], b: List[float]) -> float:
+    """Cosine similarity between two equal-length vectors."""
+    dot = sum(x * y for x, y in zip(a, b))
+    mag_a = sum(x * x for x in a) ** 0.5
+    mag_b = sum(y * y for y in b) ** 0.5
+    if mag_a == 0.0 or mag_b == 0.0:
+        return 0.0
+    return dot / (mag_a * mag_b)
+
+
+def shortlist_encodings(
+    hist: array.array,
+    total: int,
+    top_n: int = 12,
+) -> List[Tuple[str, float]]:
+    """
+    Use the pre-computed fingerprint table to rank all encodings by
+    byte-distribution similarity, returning the top_n most likely candidates.
+    O(k) where k = number of fingerprints (~99).  No decoding performed.
+    """
+    scores = list(_scores_for_hist(hist).items())
+
+    scores.sort(key=lambda x: x[1], reverse=True)
+    return scores[:top_n]
+
+
+def fingerprint_cosine_for_encoding(data: bytes, encoding: str) -> float:
+    """
+    Cosine similarity (0..1) between `data`'s byte histogram and the
+    precomputed fingerprint for `encoding`. Used to break ties when several
+    decodings look linguistically plausible (e.g. Big5 bytes mis-read as cp949).
+    """
+    try:
+        from .data.fingerprints import ENCODING_FINGERPRINTS
+    except ImportError:
+        return 0.0
+    fp = ENCODING_FINGERPRINTS.get(encoding)
+    if fp is None:
+        return 0.0
+    n = len(data)
+    if n == 0:
+        return 0.0
+    hist = byte_histogram(data)
+    ratios = histogram_to_ratios(hist, n)
+    return _cosine_similarity(ratios, fp)
+
+
+@lru_cache(maxsize=1)
+def _fingerprint_groups() -> list[tuple[list[str], tuple[float, ...]]]:
+    from .data.fingerprints import ENCODING_FINGERPRINTS
+
+    groups: dict[tuple[float, ...], list[str]] = {}
+    for encoding, raw_vector in ENCODING_FINGERPRINTS.items():
+        groups.setdefault(tuple(raw_vector), []).append(encoding)
+    result = []
+    for vector, names in groups.items():
+        norm = sum(v * v for v in vector) ** 0.5
+        result.append((names, tuple(v / norm if norm else 0.0 for v in vector)))
+    return result
+
+
+# Static profiles only: identical vectors share one dot product.
+
+
+def _scores_for_hist(hist: array.array) -> dict[str, float]:
+    norm = sum(n * n for n in hist) ** 0.5
+    if not norm:
+        return {name: 0.0 for names, _ in _fingerprint_groups() for name in names}
+    observed = [(i, n / norm) for i, n in enumerate(hist) if n]
+    scores: dict[str, float] = {}
+    for names, vector in _fingerprint_groups():
+        score = sum(n * vector[i] for i, n in observed)
+        scores.update((name, score) for name in names)
+    # Restore table order to preserve stable ties.
+    from .data.fingerprints import ENCODING_FINGERPRINTS
+
+    return {name: scores[name] for name in ENCODING_FINGERPRINTS}
+
+
+def fingerprint_scores(data: bytes) -> dict[str, float]:
+    """Score distinct static profiles from one sparse byte histogram."""
+    return _scores_for_hist(byte_histogram(data))
+
+
+from ._rust import (  # noqa: E402 — intentional late import
+    _RUST_AVAILABLE,
+    rust_byte_histogram,
+    rust_utf8_continuation_score,
+)
+
+
+def byte_histogram(data: bytes) -> array.array:
+    """
+    Compute byte frequency histogram.
+    Returns a 256-element ``array.array("Q", ...)`` of occurrence counts.
+    O(n), single pass. Uses Rust when available.
+    """
+    if _RUST_AVAILABLE:
+        return rust_byte_histogram(data)
+    return _byte_histogram_pure(data)
+
+
+def utf8_continuation_score(data: bytes) -> float:
+    """
+    Score how well `data` fits UTF-8 multibyte sequence structure.
+    Returns 0.0 (no evidence) to 1.0 (strong UTF-8 multibyte pattern).
+    Works even when the data is not fully valid UTF-8. Uses Rust when available.
+    """
+    if _RUST_AVAILABLE:
+        return rust_utf8_continuation_score(data)
+    return _utf8_continuation_score_pure(data)

bytesense/heuristics.py

@@ -0,0 +1,161 @@
+"""
+Raw-byte priors for candidate ordering (no external detectors).
+
+Heuristic ranges are approximate; they only reorder the existing shortlist so
+likely encodings are tried earlier before decode+mess ranking.
+"""
+from __future__ import annotations
+
+from typing import List, Optional
+
+
+def _bump_to_front(candidates: List[str], *want: str) -> List[str]:
+    seen: set[str] = set()
+    front: List[str] = []
+    for e in want:
+        if e in candidates and e not in seen:
+            front.append(e)
+            seen.add(e)
+    rest = [c for c in candidates if c not in seen]
+    return front + rest
+
+
+def hebrew_sbcs_likelihood(data: bytes) -> float:
+    """Share of high bytes in typical Windows-1255 Hebrew letter range (0xE0–0xFA)."""
+    if len(data) < 8:
+        return 0.0
+    hi = sum(1 for b in data if b >= 0x80)
+    if hi < 8:
+        return 0.0
+    heb = sum(1 for b in data if 0xE0 <= b <= 0xFA)
+    return heb / hi
+
+
+def thai_tis620_likelihood(data: bytes) -> float:
+    """
+    Share of high bytes in the lower TIS-620 Thai band (0xA1–0xDF).
+
+    Excludes 0xE0–0xFA, which overlaps Windows-1255 Hebrew and inflates false Thai scores.
+    """
+    if len(data) < 8:
+        return 0.0
+    hi = sum(1 for b in data if b >= 0x80)
+    if hi < 8:
+        return 0.0
+    th = sum(1 for b in data if 0xA1 <= b <= 0xDF)
+    return th / hi
+
+
+def koi8_byte_hint(data: bytes) -> bool:
+    """KOI8-R Cyrillic uses many high bytes in 0xC0–0xFF — avoid false Hebrew reorder."""
+    hi = [b for b in data if b >= 0x80]
+    if len(hi) < 24:
+        return False
+    block = sum(1 for b in hi if 0xC0 <= b <= 0xFF)
+    return block / len(hi) >= 0.58
+
+
+def cp866_vs_cp1251_hint(data: bytes) -> Optional[str]:
+    """DOS (cp866) block 0x80–0xAF vs Windows (cp1251) 0xC0–0xFF — rough split."""
+    hi = [b for b in data if b >= 0x80]
+    if len(hi) < 20:
+        return None
+    b866 = sum(1 for b in hi if 0x80 <= b <= 0xAF)
+    b1251 = sum(1 for b in hi if 0xC0 <= b <= 0xFF)
+    t = len(hi)
+    if b866 / t >= 0.40 and b1251 / t <= 0.42:
+        return "cp866"
+    if b1251 / t >= 0.40 and b866 / t <= 0.38:
+        return "cp1251"
+    return None
+
+
+def japanese_mbcs_bias(data: bytes) -> Optional[str]:
+    """Rough Shift_JIS vs EUC-JP from multi-byte pair patterns."""
+    if len(data) < 24:
+        return None
+    sj = 0
+    ej = 0
+    for i in range(len(data) - 1):
+        b1, b2 = data[i], data[i + 1]
+        if 0x81 <= b1 <= 0x9F or 0xE0 <= b1 <= 0xFC:
+            if 0x40 <= b2 <= 0xFC and b2 != 0x7F:
+                sj += 1
+        if 0xA1 <= b1 <= 0xFE and 0xA1 <= b2 <= 0xFE:
+            ej += 1
+        if b1 == 0x8E and 0xA1 <= b2 <= 0xDF:
+            ej += 2
+    if sj + ej < 14:
+        return None
+    # CP866 Cyrillic: moderate EUC-like pair counts without true JIS dominance — not Japanese.
+    hint = cp866_vs_cp1251_hint(data)
+    if hint == "cp866":
+        if ej > sj * 5.0:
+            return "euc_jp"
+        if sj > ej * 1.4 and sj >= 40:
+            return "shift_jis"
+        return None
+    if sj > ej * 1.2:
+        return "shift_jis"
+    if ej > sj * 1.2:
+        return "euc_jp"
+    return None
+
+
+def chinese_big5_vs_gb_hint(data: bytes) -> Optional[str]:
+    """
+    GB18030 can use 4-byte sequences; Big5 is overwhelmingly two-byte pairs.
+    Very rough — ranking still validates via decode+mess.
+    """
+    if len(data) < 32:
+        return None
+    quadish = 0
+    for i in range(len(data) - 3):
+        if 0x81 <= data[i] <= 0xFE and 0x30 <= data[i + 1] <= 0x39:
+            quadish += 1
+    if quadish >= max(4, len(data) // 400):
+        return "gb18030"
+    pairs = 0
+    for i in range(len(data) - 1):
+        b1, b2 = data[i], data[i + 1]
+        if 0xA1 <= b1 <= 0xFE and 0x40 <= b2 <= 0xFE:
+            pairs += 1
+    if pairs >= len(data) // 6:
+        return "big5"
+    return None
+
+
+def reorder_candidates(data: bytes, candidates: List[str]) -> List[str]:
+    """Apply raw-byte hints by moving likely encodings earlier."""
+    out = list(candidates)
+
+    if (
+        hebrew_sbcs_likelihood(data) >= 0.50
+        and cp866_vs_cp1251_hint(data) is None
+        and not koi8_byte_hint(data)
+    ):
+        out = _bump_to_front(out, "cp1255", "iso8859_8")
+
+    if thai_tis620_likelihood(data) >= 0.52 and hebrew_sbcs_likelihood(data) < 0.55:
+        out = _bump_to_front(out, "tis_620", "iso8859_11")
+
+    hint = cp866_vs_cp1251_hint(data)
+    if hint == "cp866":
+        out = _bump_to_front(out, "cp866", "koi8_r", "cp1251")
+    elif hint == "cp1251":
+        out = _bump_to_front(out, "cp1251", "koi8_r", "cp866")
+
+    jb = japanese_mbcs_bias(data)
+    if jb == "shift_jis":
+        out = _bump_to_front(out, "shift_jis", "cp932", "euc_jp", "iso2022_jp")
+    elif jb == "euc_jp":
+        out = _bump_to_front(out, "euc_jp", "iso2022_jp", "shift_jis", "cp932")
+
+    if jb is None:
+        ch = chinese_big5_vs_gb_hint(data)
+        if ch == "gb18030":
+            out = _bump_to_front(out, "gb18030", "gbk", "gb2312", "big5", "big5hkscs")
+        elif ch == "big5":
+            out = _bump_to_front(out, "big5", "big5hkscs", "gb18030", "gbk", "gb2312")
+
+    return out

bytesense/hints.py

@@ -0,0 +1,104 @@
+"""
+Encoding hint extraction from HTTP headers and HTML/XML documents.
+
+Used internally by StreamDetector, but also available as a public API
+for callers who already have the raw content and headers.
+"""
+
+from __future__ import annotations
+
+import codecs
+import re
+from typing import Optional
+
+_XML_DECL_RE = re.compile(rb'<\?xml[^>]*encoding=["\']([^"\']+)["\']', re.IGNORECASE)
+_HTML_META_RE = re.compile(rb'<meta[^>]+charset\s*=\s*["\']?\s*([a-zA-Z0-9_\-]+)', re.IGNORECASE)
+_HTTP_EQUIV_RE = re.compile(
+    rb'<meta[^>]+http-equiv\s*=\s*["\']?content-type["\']?[^>]*'
+    rb'content\s*=\s*["\']?[^"\']*charset=([a-zA-Z0-9_\-]+)',
+    re.IGNORECASE,
+)
+_HTTP_HEADER_RE = re.compile(r'charset\s*=\s*["\']?\s*([a-zA-Z0-9_\-]+)', re.IGNORECASE)
+
+
+def _normalise(enc_str: str) -> Optional[str]:
+    try:
+        return codecs.lookup(enc_str.strip()).name.replace("-", "_")
+    except LookupError:
+        return None
+
+
+def hint_from_http_headers(headers: dict[str, str]) -> Optional[str]:
+    """
+    Extract encoding hint from HTTP response headers.
+
+    Args:
+        headers: Dict of header name → value (case-insensitive matching).
+
+    Returns:
+        Normalised IANA encoding name or None.
+
+    Example::
+
+        enc = hint_from_http_headers({"Content-Type": "text/html; charset=utf-8"})
+        # → "utf_8"
+    """
+    ct = next((value for name, value in headers.items() if name.lower() == "content-type"), "")
+    m = _HTTP_HEADER_RE.search(ct)
+    if m:
+        return _normalise(m.group(1))
+    return None
+
+
+def hint_from_content(data: bytes, max_scan_bytes: int = 4096) -> Optional[str]:
+    """
+    Extract encoding hint from the first ``max_scan_bytes`` of an HTML or XML document.
+
+    Looks for:
+    - XML declaration: ``<?xml version="1.0" encoding="UTF-8"?>``
+    - HTML meta charset: ``<meta charset="utf-8">``
+    - HTML http-equiv: ``<meta http-equiv="Content-Type" content="text/html; charset=utf-8">``
+
+    Args:
+        data:           Raw bytes (need not be fully decoded).
+        max_scan_bytes: How far into the document to scan.
+
+    Returns:
+        Normalised IANA encoding name or None.
+    """
+    probe = data[:max_scan_bytes]
+    for pattern in (_XML_DECL_RE, _HTML_META_RE, _HTTP_EQUIV_RE):
+        m = pattern.search(probe)
+        if m:
+            try:
+                enc_str = m.group(1).decode("ascii", errors="ignore")
+                result = _normalise(enc_str)
+                if result:
+                    return result
+            except Exception:
+                pass
+    return None
+
+
+def best_hint(
+    data: bytes,
+    headers: Optional[dict[str, str]] = None,
+    max_scan_bytes: int = 4096,
+) -> Optional[str]:
+    """
+    Return the best encoding hint from HTTP headers and/or document content.
+
+    HTTP headers take priority (more reliable than meta tags).
+
+    Args:
+        data:    Raw document bytes.
+        headers: HTTP response headers dict (optional).
+
+    Returns:
+        Normalised IANA encoding name or None.
+    """
+    if headers:
+        h = hint_from_http_headers(headers)
+        if h:
+            return h
+    return hint_from_content(data, max_scan_bytes)

bytesense/legacy.py

@@ -0,0 +1,38 @@
+"""
+chardet / charset-normalizer drop-in compatibility layer.
+
+Migration:
+    # Before
+    from chardet import detect
+    # or
+    from charset_normalizer import detect
+
+    # After
+    from bytesense import detect   # identical interface
+"""
+from __future__ import annotations
+
+from typing import Dict, Optional, Union
+
+from .api import from_bytes
+
+
+def detect(byte_str: Union[bytes, bytearray]) -> Dict[str, Optional[object]]:
+    """
+    Drop-in replacement for ``chardet.detect()`` and
+    ``charset_normalizer.detect()``.
+
+    Args:
+        byte_str: The byte sequence to examine.
+
+    Returns:
+        ``{"encoding": str|None, "confidence": float|None, "language": str}``
+    """
+    if not isinstance(byte_str, (bytes, bytearray)):
+        raise TypeError(f"Expected bytes or bytearray, got {type(byte_str).__name__!r}")
+    result = from_bytes(bytes(byte_str))
+    return {
+        "encoding": result.encoding,
+        "confidence": result.confidence if result.encoding else None,
+        "language": result.language,
+    }