bytesense 1.0.0__py3-none-any.whl

bytesense/mess.py

@@ -0,0 +1,179 @@
+"""
+Mess / chaos detector.
+
+Scores how "garbled" a decoded string is.
+0.0 = clean human-readable text.
+1.0 = completely garbled (wrong encoding).
+
+Improvements over charset-normalizer:
+- Weighted multi-component score (not a single heuristic)
+- Bigram validity check for Latin scripts
+- Word-length plausibility check
+- Sliding-window approach for large texts
+"""
+
+from __future__ import annotations
+
+from collections import Counter
+from functools import lru_cache
+from typing import Optional, Tuple
+
+from .constant import BIGRAM_FREQUENCIES
+
+_VALID_BIGRAMS = {lang: frozenset(values) for lang, values in BIGRAM_FREQUENCIES.items()}
+_ALL_BIGRAMS = frozenset(bg for values in _VALID_BIGRAMS.values() for bg in values)
+
+
+def _cjk_ratio(text: str) -> float:
+    if not text:
+        return 0.0
+    return sum(n for c, n in Counter(text).items() if "\u4e00" <= c <= "\u9fff") / len(text)
+
+
+def _hangul_ratio(text: str) -> float:
+    if not text:
+        return 0.0
+    return sum(n for c, n in Counter(text).items() if "\uac00" <= c <= "\ud7a3") / len(text)
+
+
+def _skip_latin_mess_heuristics(text: str) -> bool:
+    """Ideographic / Hangul text: Latin bigram/word-length heuristics are misleading."""
+    if not text:
+        return False
+    return _cjk_ratio(text) > 0.18 or _hangul_ratio(text) > 0.18
+
+
+@lru_cache(maxsize=16_384)
+def _is_printable(char: str) -> bool:
+    return char.isprintable()
+
+
+def _latin_extended_fraction(text: str) -> float:
+    """Share of Latin letters that use Latin Extended-A/B (Polish, Baltic, etc.)."""
+    counts = Counter(text)
+    latin = sum(n for c, n in counts.items() if c.isalpha() and ord(c) < 0x300)
+    if latin < 4:
+        return 0.0
+    ext = sum(n for c, n in counts.items() if c.isalpha() and 0x100 <= ord(c) <= 0x024F)
+    return ext / latin
+
+
+def _bigram_mess(text: str, language_hint: Optional[str] = None) -> float:
+    """
+    Fraction of Latin bigrams that are NOT in the valid set for any language.
+    Lower = cleaner text.
+    """
+    valid = _VALID_BIGRAMS.get(language_hint or "", _ALL_BIGRAMS)
+
+    if not valid:
+        return 0.0
+
+    if _skip_latin_mess_heuristics(text):
+        return 0.0
+
+    # Latin Extended: English-centric bigrams falsely flag Polish/Czech/Baltic as noise.
+    if _latin_extended_fraction(text) > 0.22:
+        return 0.0
+
+    table = {ord(c): c.lower() if c.isalpha() and ord(c) < 0x250 else None for c in set(text)}
+    latin = text.translate(table)
+    if len(latin) < 8:
+        return 0.0
+
+    total = len(latin) - 1
+    invalid = sum(1 for i in range(total) if latin[i] + latin[i + 1] not in valid)
+    return invalid / total if total > 0 else 0.0
+
+
+def _unprintable_ratio(text: str) -> float:
+    n = len(text)
+    if n == 0:
+        return 0.0
+    bad = sum(
+        count
+        for c, count in Counter(text).items()
+        if not _is_printable(c) and c not in ("\n", "\r", "\t")
+    )
+    return bad / n
+
+
+def _suspicious_ratio(text: str) -> float:
+    n = len(text)
+    if n == 0:
+        return 0.0
+    suspicious = sum(
+        count for c, count in Counter(text).items() if c == "\ufffd" or "\ue000" <= c <= "\uf8ff"
+    )
+
+    return min(suspicious / n, 1.0)
+
+
+def _word_length_mess(text: str) -> float:
+    if _skip_latin_mess_heuristics(text):
+        return 0.0
+    words = text.split()
+    if not words:
+        return 0.0
+    avg = sum(len(w) for w in words) / len(words)
+    # Penalise: average word length > 25 characters suggests garbled text
+    return min(max(avg - 25.0, 0.0) / 25.0, 1.0)
+
+
+def mess_ratio(
+    decoded: str,
+    threshold: float = 0.2,
+    language_hint: Optional[str] = None,
+) -> float:
+    """
+    Compute chaos ratio for a decoded string chunk.
+
+    Components (weighted sum):
+      0.45 — unprintable character ratio
+      0.25 — suspicious Unicode range ratio
+      0.20 — bigram invalidity ratio  (Latin text only)
+      0.10 — word length plausibility
+    """
+    if not decoded:
+        return 0.0
+
+    ratio = (
+        _unprintable_ratio(decoded) * 0.45
+        + _suspicious_ratio(decoded) * 0.25
+        + _bigram_mess(decoded, language_hint) * 0.20
+        + _word_length_mess(decoded) * 0.10
+    )
+    return min(ratio, 1.0)
+
+
+def sliding_window_mess(
+    decoded: str,
+    window_size: int = 512,
+    threshold: float = 0.2,
+    language_hint: Optional[str] = None,
+) -> Tuple[float, bool]:
+    """
+    Compute mess ratio with a sliding window.
+    More accurate than a single pass on large or mixed-content strings.
+
+    Returns:
+        (mean_mess_ratio, exceeded_threshold_early)
+    """
+    if len(decoded) <= window_size:
+        r = mess_ratio(decoded, threshold, language_hint)
+        return r, r >= threshold
+
+    step = window_size // 2
+    ratios = []
+    exceeded = 0
+
+    for i in range(0, len(decoded) - window_size + 1, step):
+        r = mess_ratio(decoded[i : i + window_size], threshold, language_hint)
+        ratios.append(r)
+        if r >= threshold:
+            exceeded += 1
+        # Early exit: more than half the windows already exceed threshold
+        if exceeded > max(2, len(ratios) // 2):
+            return sum(ratios) / len(ratios), True
+
+    mean = sum(ratios) / len(ratios) if ratios else 0.0
+    return mean, mean >= threshold

bytesense/models.py

@@ -0,0 +1,98 @@
+from __future__ import annotations
+
+import sys
+from dataclasses import dataclass
+from typing import Dict, List, Optional, Tuple
+
+_SLOTS_KW = {"slots": True} if sys.version_info >= (3, 10) else {}
+
+
+@dataclass(frozen=True, **_SLOTS_KW)
+class EncodingAlternative:
+    """A plausible alternative encoding that did not win."""
+
+    encoding: str
+    confidence: float
+    language: str
+
+    def to_dict(self) -> Dict[str, object]:
+        return {
+            "encoding": self.encoding,
+            "confidence": self.confidence,
+            "language": self.language,
+        }
+
+
+@dataclass(**_SLOTS_KW)
+class DetectionResult:
+    """
+    Full result object returned by all bytesense detection functions.
+
+    Attributes:
+        encoding:            Python codec name, e.g. ``"utf_8"``, ``"cp1252"``.
+                             ``None`` if detection failed completely.
+        confidence:          Heuristic evidence score in 0.0–1.0, not a probability.
+        confidence_interval: Always None; heuristic scores have no statistical CI.
+        language:            Human-readable language name, e.g. ``"French"``.
+                             Empty string if not determined.
+        alternatives:        Other plausible encodings, sorted by confidence descending.
+        bom_detected:        ``True`` if a BOM/SIG was found.
+        chaos:               Control-character ratio in the scoring sample.
+        coherence:           Character-pair support (legacy) or optional language support (UTF-8).
+        why:                 Human-readable explanation of the detection decision.
+        byte_count:          Number of bytes accepted by this call or stream.
+        bytes_examined:      Bytes used for scoring or a direct fast-path decision.
+        bytes_validated:     Bytes strictly validated under the returned codec.
+        complete:            The input ended; False for previews/explicit budgets.
+        status:              matched, ambiguous, unknown, binary, or invalid.
+    """
+
+    encoding: Optional[str]
+    confidence: float
+    confidence_interval: Optional[Tuple[float, float]]
+    language: str
+    alternatives: List[EncodingAlternative]
+    bom_detected: bool
+    chaos: float
+    coherence: float
+    why: str
+    byte_count: int
+    bytes_examined: int = 0
+    bytes_validated: int = 0
+    complete: bool = False
+    status: str = "unknown"
+
+    # ------------------------------------------------------------------
+    # chardet / charset-normalizer compatibility helpers
+    # ------------------------------------------------------------------
+
+    def __str__(self) -> str:
+        return (
+            f"DetectionResult(encoding={self.encoding!r}, "
+            f"confidence={self.confidence:.3f}, "
+            f"language={self.language!r})"
+        )
+
+    def __repr__(self) -> str:
+        return self.__str__()
+
+    def __bool__(self) -> bool:
+        return self.encoding is not None
+
+    def to_dict(self) -> Dict[str, object]:
+        return {
+            "encoding": self.encoding,
+            "confidence": self.confidence,
+            "confidence_interval": None,
+            "language": self.language,
+            "alternatives": [a.to_dict() for a in self.alternatives],
+            "bom_detected": self.bom_detected,
+            "chaos": self.chaos,
+            "coherence": self.coherence,
+            "why": self.why,
+            "byte_count": self.byte_count,
+            "bytes_examined": self.bytes_examined,
+            "bytes_validated": self.bytes_validated,
+            "complete": self.complete,
+            "status": self.status,
+        }

bytesense/multi.py

@@ -0,0 +1,209 @@
+"""
+Multi-encoding document detector.
+
+Splits a byte sequence into segments and detects encoding per-segment.
+Useful for legacy email with mixed encodings or multi-part documents.
+
+"""
+
+from __future__ import annotations
+
+import codecs
+import sys
+from dataclasses import dataclass, replace
+from typing import Dict, List, Optional
+
+from .api import from_bytes
+from .models import DetectionResult
+
+_SLOTS_KW = {"slots": True} if sys.version_info >= (3, 10) else {}
+
+
+@dataclass(**_SLOTS_KW)
+class DocumentSegment:
+    """A contiguous segment of bytes with a detected encoding."""
+
+    start: int
+    end: int
+    data: bytes
+    detection: DetectionResult
+
+    @property
+    def encoding(self) -> Optional[str]:
+        return self.detection.encoding
+
+    @property
+    def text(self) -> str:
+        if self.encoding is None:
+            raise UnicodeError("segment encoding is unknown; choose an explicit decoding policy")
+        return self.data.decode(self.encoding, errors="strict")
+
+    def to_dict(self) -> Dict[str, object]:
+        return {
+            "start": self.start,
+            "end": self.end,
+            "length": self.end - self.start,
+            "encoding": self.encoding,
+            "confidence": self.detection.confidence,
+            "language": self.detection.language,
+        }
+
+
+@dataclass(**_SLOTS_KW)
+class MultiEncodingResult:
+    """Result of multi-encoding document analysis."""
+
+    segments: List[DocumentSegment]
+    is_uniform: bool  # True if all segments have the same encoding
+    dominant: Optional[str]  # Most common encoding by byte weight
+
+    @property
+    def full_text(self) -> str:
+        """Concatenate all segments decoded with their respective encodings."""
+        return "".join(seg.text for seg in self.segments)
+
+    def to_dict(self) -> Dict[str, object]:
+        return {
+            "is_uniform": self.is_uniform,
+            "dominant": self.dominant,
+            "segment_count": len(self.segments),
+            "segments": [s.to_dict() for s in self.segments],
+        }
+
+
+def detect_multi(
+    data: bytes,
+    segment_size: int = 4096,
+    min_segment_bytes: int = 128,
+    merge_threshold: float = 0.85,
+) -> MultiEncodingResult:
+    """
+    Detect encoding(s) in a potentially mixed-encoding document.
+
+    Algorithm:
+    1. Split `data` into contiguous segments near `segment_size` bytes.
+    2. Detect encoding for each segment independently.
+    3. Merge adjacent segments with the same encoding.
+    4. Return the segment list.
+
+    Args:
+        data:               Byte sequence to analyse.
+        segment_size:       Initial segment size in bytes.
+        min_segment_bytes:  Minimum bytes per segment (smaller segments are merged).
+        merge_threshold:    Confidence threshold to merge adjacent same-encoding segments.
+
+    Returns:
+        :class:`MultiEncodingResult`
+    """
+    if not isinstance(data, bytes):
+        raise TypeError("data must be bytes")
+    if not isinstance(segment_size, int) or isinstance(segment_size, bool) or segment_size <= 0:
+        raise ValueError("segment_size must be a positive integer")
+    if (
+        not isinstance(min_segment_bytes, int)
+        or isinstance(min_segment_bytes, bool)
+        or min_segment_bytes <= 0
+    ):
+        raise ValueError("min_segment_bytes must be a positive integer")
+    if not 0.0 <= merge_threshold <= 1.0:
+        raise ValueError("merge_threshold must be between 0 and 1")
+    whole = from_bytes(data)
+    # A fully validated Unicode/stateful stream must not be cut into arbitrary
+    # byte windows; those windows can start inside a code unit or shift state.
+    uniform = whole.encoding is not None and (
+        whole.encoding.startswith("utf_")
+        or (whole.encoding.startswith("iso2022_") or whole.encoding == "hz")
+    )
+    if len(data) <= segment_size or uniform:
+        # Single-segment case — fast path
+        result = whole
+        seg = DocumentSegment(start=0, end=len(data), data=data, detection=result)
+        return MultiEncodingResult(
+            segments=[seg],
+            is_uniform=True,
+            dominant=result.encoding,
+        )
+
+    # Detect per-segment
+    raw_segments: list[tuple[int, int, DetectionResult]] = []
+    pos = 0
+    while pos < len(data):
+        end = min(pos + segment_size, len(data))
+        if 0 < len(data) - end < min_segment_bytes:
+            end = len(data)
+        # When a full-file candidate is available, avoid cutting a complete
+        # multibyte character at the right edge. Internal decode failures are
+        # left to segment detection; no bytes are discarded.
+        if whole.encoding and end < len(data):
+            try:
+                decoder = codecs.getincrementaldecoder(whole.encoding)()
+                decoder.decode(data[pos:end], final=False)
+                for _ in range(8):
+                    pending = decoder.getstate()[0]
+                    if not pending or end == len(data):
+                        break
+                    decoder.decode(data[end : end + 1], final=False)
+                    end += 1
+            except (UnicodeError, LookupError, TypeError):
+                pass
+        chunk = data[pos:end]
+        r = from_bytes(chunk)
+        raw_segments.append((pos, end, r))
+        pos = end
+
+    # Merge adjacent segments with same encoding
+    merged: list[DocumentSegment] = []
+    if raw_segments:
+        cur_start, cur_end, cur_result = raw_segments[0]
+        for start, end, result in raw_segments[1:]:
+            if (
+                result.encoding == cur_result.encoding
+                and result.confidence >= merge_threshold
+                and cur_result.confidence >= merge_threshold
+            ):
+                cur_end = end
+                merged_len = cur_end - cur_start
+                cur_result = replace(
+                    cur_result,
+                    byte_count=merged_len,
+                    bytes_validated=merged_len if cur_result.encoding else 0,
+                    bytes_examined=cur_result.bytes_examined + result.bytes_examined,
+                    confidence=min(cur_result.confidence, result.confidence),
+                )
+            else:
+                merged.append(
+                    DocumentSegment(
+                        start=cur_start,
+                        end=cur_end,
+                        data=data[cur_start:cur_end],
+                        detection=cur_result,
+                    )
+                )
+                cur_start, cur_end, cur_result = start, end, result
+        merged.append(
+            DocumentSegment(
+                start=cur_start,
+                end=cur_end,
+                data=data[cur_start:cur_end],
+                detection=cur_result,
+            )
+        )
+
+    if not merged and data:
+        r = from_bytes(data)
+        merged = [DocumentSegment(start=0, end=len(data), data=data, detection=r)]
+
+    # Determine dominant encoding by byte weight
+    enc_weights: dict[str, int] = {}
+    for seg in merged:
+        enc = seg.encoding or "unknown"
+        enc_weights[enc] = enc_weights.get(enc, 0) + (seg.end - seg.start)
+    dominant = max(enc_weights, key=lambda k: enc_weights[k]) if enc_weights else None
+
+    is_uniform = len({s.encoding for s in merged}) <= 1
+
+    return MultiEncodingResult(
+        segments=merged,
+        is_uniform=is_uniform,
+        dominant=dominant,
+    )