glitchlings 1.0.0__cp313-cp313-win_amd64.whl

glitchlings/zoo/scannequin.py

@@ -0,0 +1,370 @@
+"""Scannequin: Research-backed OCR simulation glitchling.
+
+This module provides OCR-style text corruption based on empirical research
+into document degradation and character recognition failures.
+
+References
+----------
+- Kolak & Resnik (2002) - Noisy-channel OCR error modeling
+- Kanungo et al. (1994) - "Nonlinear Global and Local Document Degradation Models"
+  https://kanungo.com/pubs/ijist94-model.pdf
+- Li, Lopresti, Nagy, Tompkins (1996) - "Validation of Image Defect Models for OCR"
+  https://sites.ecse.rpi.edu/~nagy/PDF_files/Li_Lopresti_Tompkins_PAMI96.pdf
+- Rice et al. / UNLV-ISRI Annual Tests (1995) - Quality preset empirical basis
+- Taghva et al. - "Context beats Confusion"
+  https://www.projectcomputing.com/resources/CorrectingNoisyOCR.pdf
+- ICDAR Robust Reading Competitions
+  https://dl.acm.org/doi/abs/10.1007/s10032-004-0134-3
+- Smith (2007) - Tesseract OCR architecture
+"""
+
+import random
+from typing import Any, Literal, cast
+
+from glitchlings.constants import (
+    DEFAULT_SCANNEQUIN_BIAS_BETA,
+    DEFAULT_SCANNEQUIN_BIAS_K,
+    DEFAULT_SCANNEQUIN_BURST_ENTER,
+    DEFAULT_SCANNEQUIN_BURST_EXIT,
+    DEFAULT_SCANNEQUIN_BURST_MULTIPLIER,
+    DEFAULT_SCANNEQUIN_RATE,
+    DEFAULT_SCANNEQUIN_SPACE_DROP_RATE,
+    DEFAULT_SCANNEQUIN_SPACE_INSERT_RATE,
+    SCANNEQUIN_PRESETS,
+)
+from glitchlings.internal.rust_ffi import ocr_artifacts_rust, resolve_seed
+
+from .core import AttackOrder, AttackWave, Glitchling, PipelineOperationPayload
+
+# Type alias for preset names
+PresetName = Literal["clean_300dpi", "newspaper", "fax", "photocopy_3rd_gen"]
+
+
+def ocr_artifacts(
+    text: str,
+    rate: float | None = None,
+    seed: int | None = None,
+    rng: random.Random | None = None,
+    *,
+    burst_enter: float | None = None,
+    burst_exit: float | None = None,
+    burst_multiplier: float | None = None,
+    bias_k: int | None = None,
+    bias_beta: float | None = None,
+    space_drop_rate: float | None = None,
+    space_insert_rate: float | None = None,
+) -> str:
+    """Introduce OCR-like artifacts into text with research-backed enhancements.
+
+    This function simulates OCR errors using three research-backed features:
+
+    **Burst Model (Kanungo et al., 1994)**
+    Real document defects are spatially correlated - a coffee stain or fold
+    affects a region, not individual characters. Uses an HMM to create error
+    clusters simulating smudges, folds, or degraded scan regions.
+
+    **Document-Level Bias (UNLV-ISRI, 1995)**
+    Documents scanned under the same conditions exhibit consistent error
+    profiles. Randomly selects K confusion patterns and amplifies their
+    selection probability, creating "why does it always turn 'l' into '1'"
+    consistency.
+
+    **Whitespace Errors (Smith, 2007; ICDAR)**
+    Models OCR segmentation failures that cause word merges/splits. These
+    happen before character recognition in the real pipeline.
+
+    Parameters
+    ----------
+    text : str
+        Input text to corrupt.
+    rate : float, optional
+        Base probability of applying a confusion to any given candidate.
+        Default is 0.02.
+    seed : int, optional
+        Deterministic seed for reproducibility.
+    rng : random.Random, optional
+        Optional RNG for seed generation.
+    burst_enter : float, optional
+        Probability of transitioning from clean to harsh state (default 0.0).
+    burst_exit : float, optional
+        Probability of transitioning from harsh to clean state (default 0.3).
+    burst_multiplier : float, optional
+        Rate multiplier when in harsh state (default 3.0).
+    bias_k : int, optional
+        Number of confusion patterns to amplify per document (default 0).
+    bias_beta : float, optional
+        Amplification factor for selected patterns (default 2.0).
+    space_drop_rate : float, optional
+        Probability of deleting a space, merging words (default 0.0).
+    space_insert_rate : float, optional
+        Probability of inserting a spurious space (default 0.0).
+
+    Returns
+    -------
+    str
+        Text with simulated OCR errors.
+
+    References
+    ----------
+    - Kanungo et al. (1994) - "Nonlinear Global and Local Document Degradation Models"
+    - Rice et al. / UNLV-ISRI Annual Tests (1995)
+    - Smith (2007) - Tesseract OCR architecture
+    - ICDAR Robust Reading Competitions
+    """
+    if not text:
+        return text
+
+    effective_rate = DEFAULT_SCANNEQUIN_RATE if rate is None else rate
+    clamped_rate = max(0.0, effective_rate)
+
+    return ocr_artifacts_rust(
+        text,
+        clamped_rate,
+        resolve_seed(seed, rng),
+        burst_enter=burst_enter,
+        burst_exit=burst_exit,
+        burst_multiplier=burst_multiplier,
+        bias_k=bias_k,
+        bias_beta=bias_beta,
+        space_drop_rate=space_drop_rate,
+        space_insert_rate=space_insert_rate,
+    )
+
+
+class Scannequin(Glitchling):
+    """Glitchling that simulates OCR artifacts with research-backed enhancements.
+
+    Scannequin introduces OCR-inspired transcription mistakes to emulate noisy
+    document scans. It now operates at **document level** to enable document-wide
+    consistency in error patterns.
+
+    Features
+    --------
+
+    **Burst Model (Kanungo et al., 1994)**
+    Uses an HMM with clean/harsh states to create spatially correlated error
+    clusters, simulating physical defects like smudges, folds, or scan artifacts.
+
+    **Document-Level Bias (UNLV-ISRI, 1995)**
+    Selects K confusion patterns at document start and amplifies their selection
+    probability, creating consistent error profiles across the document.
+
+    **Whitespace Errors (Smith, 2007; ICDAR)**
+    Models OCR segmentation failures: space drops (word merges) and spurious
+    space insertions (word splits).
+
+    **Quality Presets**
+    Based on UNLV-ISRI test regimes:
+    - ``"clean_300dpi"``: Minimal errors, good quality baseline
+    - ``"newspaper"``: Moderate errors with some burst
+    - ``"fax"``: High errors, strong burst, heavy l/1/I confusion
+    - ``"photocopy_3rd_gen"``: Very degraded, long burst runs
+
+    Parameters
+    ----------
+    rate : float, optional
+        Base probability of applying a confusion (default 0.02).
+    seed : int, optional
+        Deterministic seed.
+    preset : str, optional
+        Quality preset name. Overrides individual parameters when set.
+    burst_enter : float, optional
+        P(clean → harsh) state transition (default 0.0 = disabled).
+    burst_exit : float, optional
+        P(harsh → clean) state transition (default 0.3).
+    burst_multiplier : float, optional
+        Rate multiplier in harsh state (default 3.0).
+    bias_k : int, optional
+        Number of patterns to amplify per document (default 0 = disabled).
+    bias_beta : float, optional
+        Amplification factor for biased patterns (default 2.0).
+    space_drop_rate : float, optional
+        P(delete space, merge words) (default 0.0 = disabled).
+    space_insert_rate : float, optional
+        P(insert spurious space) (default 0.0 = disabled).
+    **kwargs
+        Additional parameters passed to base Glitchling.
+
+    Examples
+    --------
+    Basic usage with default parameters:
+
+    >>> scan = Scannequin(rate=0.02, seed=42)
+    >>> scan("The cat sat on the mat")
+    'The cat sat on the rnat'
+
+    Using a quality preset:
+
+    >>> fax_scan = Scannequin(preset="fax", seed=42)
+    >>> fax_scan("Hello world, this is a test document.")
+    'He1lo vvorld, thls is a testdocument.'
+
+    Enabling burst mode for realistic degradation:
+
+    >>> degraded = Scannequin(rate=0.03, burst_enter=0.1, burst_exit=0.2, seed=42)
+    >>> degraded("Some regions will have clustered errors like smudges.")
+    'Sorne regions will have dustered errors Iike srnudges.'
+
+    References
+    ----------
+    - Kolak & Resnik (2002) - Noisy-channel OCR error modeling
+    - Kanungo et al. (1994) - "Nonlinear Global and Local Document Degradation Models"
+    - Li, Lopresti, Nagy, Tompkins (1996) - "Validation of Image Defect Models for OCR"
+    - Rice et al. / UNLV-ISRI Annual Tests (1995)
+    - Smith (2007) - Tesseract OCR architecture
+    """
+
+    flavor = "Isn't it weird how the word 'bed' looks like a bed?"
+
+    def __init__(
+        self,
+        *,
+        rate: float | None = None,
+        seed: int | None = None,
+        preset: PresetName | None = None,
+        burst_enter: float | None = None,
+        burst_exit: float | None = None,
+        burst_multiplier: float | None = None,
+        bias_k: int | None = None,
+        bias_beta: float | None = None,
+        space_drop_rate: float | None = None,
+        space_insert_rate: float | None = None,
+        **kwargs: Any,
+    ) -> None:
+        # If preset is specified, load parameters from it
+        if preset is not None:
+            if preset not in SCANNEQUIN_PRESETS:
+                valid_presets = ", ".join(sorted(SCANNEQUIN_PRESETS.keys()))
+                msg = f"Unknown preset '{preset}'. Valid presets: {valid_presets}"
+                raise ValueError(msg)
+
+            (
+                preset_rate,
+                preset_burst_enter,
+                preset_burst_exit,
+                preset_burst_multiplier,
+                preset_bias_k,
+                preset_bias_beta,
+                preset_space_drop_rate,
+                preset_space_insert_rate,
+            ) = SCANNEQUIN_PRESETS[preset]
+
+            # Preset values are used as defaults, explicit params override
+            if rate is None:
+                rate = preset_rate
+            if burst_enter is None:
+                burst_enter = preset_burst_enter
+            if burst_exit is None:
+                burst_exit = preset_burst_exit
+            if burst_multiplier is None:
+                burst_multiplier = preset_burst_multiplier
+            if bias_k is None:
+                bias_k = preset_bias_k
+            if bias_beta is None:
+                bias_beta = preset_bias_beta
+            if space_drop_rate is None:
+                space_drop_rate = preset_space_drop_rate
+            if space_insert_rate is None:
+                space_insert_rate = preset_space_insert_rate
+
+        # Apply defaults for any remaining None values
+        effective_rate = DEFAULT_SCANNEQUIN_RATE if rate is None else rate
+        effective_burst_enter = (
+            DEFAULT_SCANNEQUIN_BURST_ENTER if burst_enter is None else burst_enter
+        )
+        effective_burst_exit = DEFAULT_SCANNEQUIN_BURST_EXIT if burst_exit is None else burst_exit
+        effective_burst_multiplier = (
+            DEFAULT_SCANNEQUIN_BURST_MULTIPLIER if burst_multiplier is None else burst_multiplier
+        )
+        effective_bias_k = DEFAULT_SCANNEQUIN_BIAS_K if bias_k is None else bias_k
+        effective_bias_beta = DEFAULT_SCANNEQUIN_BIAS_BETA if bias_beta is None else bias_beta
+        effective_space_drop_rate = (
+            DEFAULT_SCANNEQUIN_SPACE_DROP_RATE if space_drop_rate is None else space_drop_rate
+        )
+        effective_space_insert_rate = (
+            DEFAULT_SCANNEQUIN_SPACE_INSERT_RATE if space_insert_rate is None else space_insert_rate
+        )
+
+        super().__init__(
+            name="Scannequin",
+            corruption_function=ocr_artifacts,
+            # Changed from CHARACTER to DOCUMENT for document-wide consistency
+            scope=AttackWave.DOCUMENT,
+            order=AttackOrder.LATE,
+            seed=seed,
+            rate=effective_rate,
+            burst_enter=effective_burst_enter,
+            burst_exit=effective_burst_exit,
+            burst_multiplier=effective_burst_multiplier,
+            bias_k=effective_bias_k,
+            bias_beta=effective_bias_beta,
+            space_drop_rate=effective_space_drop_rate,
+            space_insert_rate=effective_space_insert_rate,
+            **kwargs,
+        )
+
+        # Store preset name if used
+        self._preset = preset
+
+    @classmethod
+    def from_preset(cls, preset: PresetName, *, seed: int | None = None) -> "Scannequin":
+        """Create a Scannequin instance from a named quality preset.
+
+        Parameters
+        ----------
+        preset : str
+            Quality preset name. One of:
+            - ``"clean_300dpi"``: Clean 300 DPI scan, minimal errors
+            - ``"newspaper"``: Newspaper-quality scan, moderate degradation
+            - ``"fax"``: Fax-quality, high error rate with l/1/I confusion
+            - ``"photocopy_3rd_gen"``: Third-generation photocopy, severe degradation
+        seed : int, optional
+            Deterministic seed for reproducibility.
+
+        Returns
+        -------
+        Scannequin
+            Configured Scannequin instance.
+
+        Examples
+        --------
+        >>> fax = Scannequin.from_preset("fax", seed=42)
+        >>> fax("The quick brown fox")
+        'Tbe quick brovvn fox'
+        """
+        return cls(preset=preset, seed=seed)
+
+    def pipeline_operation(self) -> PipelineOperationPayload:
+        """Return the Rust pipeline descriptor with all OCR parameters."""
+        rate_value = self.kwargs.get("rate", DEFAULT_SCANNEQUIN_RATE)
+        rate = DEFAULT_SCANNEQUIN_RATE if rate_value is None else float(rate_value)
+
+        return cast(
+            PipelineOperationPayload,
+            {
+                "type": "ocr",
+                "rate": rate,
+                "burst_enter": float(
+                    self.kwargs.get("burst_enter", DEFAULT_SCANNEQUIN_BURST_ENTER)
+                ),
+                "burst_exit": float(self.kwargs.get("burst_exit", DEFAULT_SCANNEQUIN_BURST_EXIT)),
+                "burst_multiplier": float(
+                    self.kwargs.get("burst_multiplier", DEFAULT_SCANNEQUIN_BURST_MULTIPLIER)
+                ),
+                "bias_k": int(self.kwargs.get("bias_k", DEFAULT_SCANNEQUIN_BIAS_K)),
+                "bias_beta": float(self.kwargs.get("bias_beta", DEFAULT_SCANNEQUIN_BIAS_BETA)),
+                "space_drop_rate": float(
+                    self.kwargs.get("space_drop_rate", DEFAULT_SCANNEQUIN_SPACE_DROP_RATE)
+                ),
+                "space_insert_rate": float(
+                    self.kwargs.get("space_insert_rate", DEFAULT_SCANNEQUIN_SPACE_INSERT_RATE)
+                ),
+            },
+        )
+
+
+# Default instance for convenience
+scannequin = Scannequin()
+
+
+__all__ = ["Scannequin", "scannequin", "ocr_artifacts", "SCANNEQUIN_PRESETS"]

glitchlings/zoo/transforms.py

@@ -0,0 +1,331 @@
+"""Pure text transformation functions.
+
+This module contains text manipulation functions that are:
+- **Pure**: Output depends only on inputs, no side effects
+- **Deterministic**: Same inputs always produce same outputs
+- **Self-contained**: No RNG, no Rust FFI, no config loading
+
+These functions receive pre-validated inputs from boundary layers
+(see validation.py) and trust that inputs are already checked.
+Core transformation code should NOT re-validate parameters.
+
+Design Philosophy
+-----------------
+This module implements the innermost layer of the purity architecture:
+
+    CLI/API → validation.py → transforms.py → Rust FFI
+    (boundary)   (boundary)     (pure core)    (impure)
+
+Functions here should:
+- Accept concrete types (not Optional unless semantically required)
+- Not log, print, or mutate external state
+- Not import impure modules (internal.rust, config loaders, etc.)
+- Document any preconditions callers must satisfy
+
+See AGENTS.md "Functional Purity Architecture" for full details.
+"""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Iterable, Mapping, Sequence
+from dataclasses import dataclass
+from typing import TypeVar, cast
+
+# ---------------------------------------------------------------------------
+# Text Tokenization
+# ---------------------------------------------------------------------------
+
+_WORD_SPLIT_PATTERN = re.compile(r"(\s+)")
+_TOKEN_EDGES_PATTERN = re.compile(r"^(\W*)(.*?)(\W*)$", re.DOTALL)
+
+
+def split_preserving_whitespace(text: str) -> list[str]:
+    """Split text while keeping whitespace tokens for stable reconstruction.
+
+    Returns alternating [word, whitespace, word, whitespace, ...] tokens.
+    Joining the result reconstructs the original text exactly.
+
+    Args:
+        text: Input text to tokenize.
+
+    Returns:
+        List of tokens alternating between non-whitespace and whitespace.
+
+    Example:
+        >>> split_preserving_whitespace("hello  world")
+        ['hello', '  ', 'world']
+    """
+    return _WORD_SPLIT_PATTERN.split(text)
+
+
+def split_token_edges(token: str) -> tuple[str, str, str]:
+    """Decompose a token into leading punctuation, core, and trailing punctuation.
+
+    Args:
+        token: A non-whitespace token.
+
+    Returns:
+        Tuple of (prefix, core, suffix) where:
+        - prefix: leading non-word characters
+        - core: central word characters
+        - suffix: trailing non-word characters
+
+    Example:
+        >>> split_token_edges('"Hello!"')
+        ('"', 'Hello', '!"')
+    """
+    match = cast(re.Match[str], _TOKEN_EDGES_PATTERN.match(token))
+    prefix, core, suffix = match.groups()
+    return prefix, core, suffix
+
+
+def compute_core_length(token: str) -> int:
+    """Compute the effective length of a token's core for weighting heuristics.
+
+    Used by weighted sampling algorithms to prioritize longer words.
+    Always returns at least 1 to avoid zero-weight issues.
+
+    Args:
+        token: A non-whitespace token.
+
+    Returns:
+        Positive integer representing the token's effective length.
+    """
+    _, core, _ = split_token_edges(token)
+    if core:
+        return len(core)
+    stripped = token.strip()
+    if stripped:
+        return len(stripped)
+    if token:
+        return len(token)
+    return 1
+
+
+@dataclass(frozen=True)
+class WordToken:
+    """Metadata describing a non-whitespace token from text tokenization.
+
+    Attributes:
+        index: Position in the parent token sequence.
+        prefix: Leading non-word characters (punctuation).
+        core: Central word characters.
+        suffix: Trailing non-word characters (punctuation).
+        core_length: Effective length for weighting (always >= 1).
+    """
+
+    index: int
+    prefix: str
+    core: str
+    suffix: str
+    core_length: int
+
+    @property
+    def has_core(self) -> bool:
+        """Return True when the token contains at least one core character."""
+        return bool(self.core)
+
+
+def collect_word_tokens(
+    tokens: Sequence[str],
+    *,
+    skip_first_word: bool = False,
+) -> list[WordToken]:
+    """Extract structured metadata for non-whitespace tokens.
+
+    Args:
+        tokens: Token sequence from split_preserving_whitespace.
+        skip_first_word: If True, exclude the first content token
+            (useful for preserving leading words in delete operations).
+
+    Returns:
+        List of WordToken instances for each non-whitespace token.
+    """
+    start = 2 if skip_first_word else 0
+    collected: list[WordToken] = []
+
+    for index in range(start, len(tokens), 2):
+        token = tokens[index]
+        if not token or token.isspace():
+            continue
+
+        prefix, core, suffix = split_token_edges(token)
+        core_length = compute_core_length(token)
+
+        collected.append(
+            WordToken(
+                index=index,
+                prefix=prefix,
+                core=core,
+                suffix=suffix,
+                core_length=core_length,
+            )
+        )
+
+    return collected
+
+
+def reassemble_tokens(tokens: Sequence[str]) -> str:
+    """Join tokens back into text, preserving original structure.
+
+    Args:
+        tokens: Token sequence (typically modified from split_preserving_whitespace).
+
+    Returns:
+        Reassembled text string.
+    """
+    return "".join(tokens)
+
+
+# ---------------------------------------------------------------------------
+# String Difference Computation
+# ---------------------------------------------------------------------------
+
+
+def compute_string_diffs(
+    original: str,
+    modified: str,
+) -> list[list[tuple[str, str, str]]]:
+    """Compare two strings and return grouped adjacent change operations.
+
+    Uses difflib's SequenceMatcher to identify changes between strings.
+    Consecutive changes are grouped together; equal regions are skipped.
+
+    Args:
+        original: The original string.
+        modified: The modified string.
+
+    Returns:
+        List of change groups. Each group is a list of (tag, old_text, new_text)
+        tuples where tag is 'replace', 'delete', or 'insert'.
+
+    Example:
+        >>> compute_string_diffs("hello world", "helo worlds")
+        [[('delete', 'l', '')], [('replace', '', 's')]]
+    """
+    import difflib
+
+    sm = difflib.SequenceMatcher(None, original, modified)
+    ops: list[list[tuple[str, str, str]]] = []
+    buffer: list[tuple[str, str, str]] = []
+
+    for tag, i1, i2, j1, j2 in sm.get_opcodes():
+        if tag == "equal":
+            if buffer:
+                ops.append(buffer)
+                buffer = []
+            continue
+        buffer.append((tag, original[i1:i2], modified[j1:j2]))
+
+    if buffer:
+        ops.append(buffer)
+
+    return ops
+
+
+# ---------------------------------------------------------------------------
+# Sequence Operations
+# ---------------------------------------------------------------------------
+
+T = TypeVar("T")
+
+
+def stable_deduplicate(items: Iterable[T]) -> list[T]:
+    """Remove duplicates while preserving original order.
+
+    Args:
+        items: Iterable of hashable items.
+
+    Returns:
+        List with duplicates removed, first occurrence preserved.
+
+    Example:
+        >>> stable_deduplicate([3, 1, 4, 1, 5, 9, 2, 6, 5])
+        [3, 1, 4, 5, 9, 2, 6]
+    """
+    seen: set[T] = set()
+    result: list[T] = []
+    for item in items:
+        if item not in seen:
+            seen.add(item)
+            result.append(item)
+    return result
+
+
+def interleave_lists(
+    primary: Sequence[T],
+    secondary: Sequence[T],
+    *,
+    secondary_first: bool = False,
+) -> list[T]:
+    """Interleave two sequences, padding shorter with empty slots.
+
+    Args:
+        primary: First sequence.
+        secondary: Second sequence.
+        secondary_first: If True, start with secondary element.
+
+    Returns:
+        Interleaved list [p0, s0, p1, s1, ...] or [s0, p0, s1, p1, ...].
+    """
+    result: list[T] = []
+    max_len = max(len(primary), len(secondary))
+
+    for i in range(max_len):
+        if secondary_first:
+            if i < len(secondary):
+                result.append(secondary[i])
+            if i < len(primary):
+                result.append(primary[i])
+        else:
+            if i < len(primary):
+                result.append(primary[i])
+            if i < len(secondary):
+                result.append(secondary[i])
+
+    return result
+
+
+# ---------------------------------------------------------------------------
+# Mapping Helpers
+# ---------------------------------------------------------------------------
+
+
+def invert_mapping(
+    mapping: Mapping[str, Sequence[str]],
+) -> dict[str, str]:
+    """Invert a one-to-many mapping into a many-to-one lookup.
+
+    Given {key: [val1, val2]}, returns {val1: key, val2: key}.
+    Later keys overwrite earlier ones if values collide.
+
+    Args:
+        mapping: Dictionary mapping keys to sequences of values.
+
+    Returns:
+        Inverted dictionary mapping each value to its key.
+    """
+    inverted: dict[str, str] = {}
+    for key, values in mapping.items():
+        for value in values:
+            inverted[value] = key
+    return inverted
+
+
+__all__ = [
+    # Tokenization
+    "split_preserving_whitespace",
+    "split_token_edges",
+    "compute_core_length",
+    "WordToken",
+    "collect_word_tokens",
+    "reassemble_tokens",
+    # Diffs
+    "compute_string_diffs",
+    # Sequences
+    "stable_deduplicate",
+    "interleave_lists",
+    # Mappings
+    "invert_mapping",
+]