glitchlings 0.10.2__cp312-cp312-macosx_11_0_universal2.whl

glitchlings/attack/encode.py

@@ -0,0 +1,114 @@
+"""Pure encoding utilities for tokenization.
+
+This module contains pure functions for encoding text using tokenizers.
+The functions here do not resolve tokenizers or perform IO - they operate
+on already-resolved Tokenizer instances.
+
+Pure guarantees:
+- No import side effects beyond stdlib
+- No file IO or network calls
+- No environment variable access
+- Deterministic output for given inputs
+
+The impure tokenizer resolution lives in tokenization.py.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Sequence
+
+if TYPE_CHECKING:  # pragma: no cover - typing only
+    from .tokenization import Tokenizer
+
+
+def encode_single(
+    tokenizer: "Tokenizer",
+    text: str,
+) -> tuple[list[str], list[int]]:
+    """Encode a single text string into tokens and IDs.
+
+    This is a thin wrapper that ensures list output types.
+
+    Args:
+        tokenizer: A resolved tokenizer instance.
+        text: Text to encode.
+
+    Returns:
+        Tuple of (tokens, token_ids) as lists.
+    """
+    tokens, ids = tokenizer.encode(text)
+    return list(tokens), list(ids)
+
+
+def encode_batch(
+    tokenizer: "Tokenizer",
+    texts: Sequence[str],
+) -> tuple[list[list[str]], list[list[int]]]:
+    """Encode multiple texts into batched tokens and IDs.
+
+    Attempts to use the tokenizer's batch_encode method if available,
+    otherwise falls back to per-item encoding.
+
+    Args:
+        tokenizer: A resolved tokenizer instance.
+        texts: Sequence of texts to encode.
+
+    Returns:
+        Tuple of (token_batches, id_batches) as nested lists.
+    """
+    # Try batch encoding if available
+    batch_encode = getattr(tokenizer, "encode_batch", None)
+    if callable(batch_encode):
+        encoded = batch_encode(texts)
+        token_batches: list[list[str]] = []
+        id_batches: list[list[int]] = []
+        for tokens, ids in encoded:
+            token_batches.append(list(tokens))
+            id_batches.append(list(ids))
+        return token_batches, id_batches
+
+    # Fallback: encode each text individually
+    token_batches_fallback: list[list[str]] = []
+    id_batches_fallback: list[list[int]] = []
+    for entry in texts:
+        tokens, ids = encode_single(tokenizer, entry)
+        token_batches_fallback.append(tokens)
+        id_batches_fallback.append(ids)
+    return token_batches_fallback, id_batches_fallback
+
+
+def describe_tokenizer(
+    tokenizer: "Tokenizer",
+    raw_spec: "str | Tokenizer | None",
+) -> str:
+    """Generate a human-readable description of a tokenizer.
+
+    Args:
+        tokenizer: The resolved tokenizer instance.
+        raw_spec: The original specification used to create/resolve the tokenizer.
+
+    Returns:
+        A descriptive string identifying the tokenizer.
+    """
+    # If the raw spec was a string, use it directly
+    if isinstance(raw_spec, str):
+        return raw_spec
+
+    # Try to get a name attribute
+    name = getattr(tokenizer, "name", None)
+    if isinstance(name, str) and name:
+        return name
+
+    # For None spec, use the class name
+    if raw_spec is None:
+        return tokenizer.__class__.__name__
+
+    # Fallback to string representation
+    return str(raw_spec)
+
+
+__all__ = [
+    "describe_tokenizer",
+    "encode_batch",
+    "encode_single",
+]

glitchlings/attack/metrics.py

@@ -0,0 +1,218 @@
+from __future__ import annotations
+
+import importlib
+from enum import Enum
+from typing import TYPE_CHECKING, Any, Protocol, cast
+
+from .metrics_dispatch import TokenBatch, TokenSequence, is_batch, validate_batch_consistency
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
+
+
+class Metric(Protocol):
+    def __call__(
+        self,
+        original_tokens: TokenSequence | TokenBatch,
+        corrupted_tokens: TokenSequence | TokenBatch,
+    ) -> float | list[float]: ...
+
+
+class BatchMetric(Protocol):
+    def __call__(self, inputs: TokenBatch, outputs: TokenBatch) -> list[float]: ...
+
+
+try:
+    _rust: Any = importlib.import_module("glitchlings._zoo_rust")
+except ModuleNotFoundError as exc:  # pragma: no cover - runtime guard
+    raise ImportError(
+        "Could not import compiled Rust extension. "
+        "Please ensure the project is installed with the Rust extension built."
+    ) from exc
+
+_single_jsd = cast(Metric, getattr(_rust, "jensen_shannon_divergence"))
+_single_ned = cast(Metric, getattr(_rust, "normalized_edit_distance"))
+_single_sr = cast(Metric, getattr(_rust, "subsequence_retention"))
+_single_ed = cast(Metric, getattr(_rust, "entropy_delta"))
+_single_msi = cast(Metric, getattr(_rust, "merge_split_index"))
+_batch_jsd = cast(BatchMetric, getattr(_rust, "batch_jensen_shannon_divergence"))
+_batch_ned = cast(BatchMetric, getattr(_rust, "batch_normalized_edit_distance"))
+_batch_sr = cast(BatchMetric, getattr(_rust, "batch_subsequence_retention"))
+_batch_ed = cast(BatchMetric, getattr(_rust, "batch_entropy_delta"))
+_batch_msi = cast(BatchMetric, getattr(_rust, "batch_merge_split_index"))
+
+
+def _dispatch_metric(
+    original: TokenSequence | TokenBatch,
+    corrupted: TokenSequence | TokenBatch,
+    *,
+    single: Metric,
+    batch: BatchMetric,
+    name: str,
+) -> float | list[float]:
+    """Dispatch metric computation to single or batch implementation.
+
+    Uses the pure is_batch function to determine which implementation to call.
+    """
+    validate_batch_consistency(original, corrupted, name)
+
+    if is_batch(original):
+        return batch(original, corrupted)
+
+    return single(original, corrupted)
+
+
+def jensen_shannon_divergence(
+    original_tokens: TokenSequence | TokenBatch,
+    corrupted_tokens: TokenSequence | TokenBatch,
+) -> float | list[float]:
+    return _dispatch_metric(
+        original_tokens,
+        corrupted_tokens,
+        single=_single_jsd,
+        batch=_batch_jsd,
+        name="jensen_shannon_divergence",
+    )
+
+
+def normalized_edit_distance(
+    original_tokens: TokenSequence | TokenBatch,
+    corrupted_tokens: TokenSequence | TokenBatch,
+) -> float | list[float]:
+    return _dispatch_metric(
+        original_tokens,
+        corrupted_tokens,
+        single=_single_ned,
+        batch=_batch_ned,
+        name="normalized_edit_distance",
+    )
+
+
+def subsequence_retention(
+    original_tokens: TokenSequence | TokenBatch,
+    corrupted_tokens: TokenSequence | TokenBatch,
+) -> float | list[float]:
+    return _dispatch_metric(
+        original_tokens,
+        corrupted_tokens,
+        single=_single_sr,
+        batch=_batch_sr,
+        name="subsequence_retention",
+    )
+
+
+def entropy_delta(
+    original_tokens: TokenSequence | TokenBatch,
+    corrupted_tokens: TokenSequence | TokenBatch,
+) -> float | list[float]:
+    """Compute normalized entropy delta between original and corrupted tokens.
+
+    Measures the change in token distribution entropy:
+    ΔH = H(corrupted) - H(original), normalized to [-1, 1].
+
+    Positive values indicate the corrupted text has higher entropy
+    (more uniform/diverse token distribution). Negative values indicate
+    lower entropy (more concentrated distribution).
+
+    Args:
+        original_tokens: Original token sequence(s).
+        corrupted_tokens: Corrupted token sequence(s).
+
+    Returns:
+        Normalized entropy delta in [-1, 1], or list for batches.
+    """
+    return _dispatch_metric(
+        original_tokens,
+        corrupted_tokens,
+        single=_single_ed,
+        batch=_batch_ed,
+        name="entropy_delta",
+    )
+
+
+def merge_split_index(
+    original_tokens: TokenSequence | TokenBatch,
+    corrupted_tokens: TokenSequence | TokenBatch,
+) -> float | list[float]:
+    """Compute merge-split index measuring subword restructuring.
+
+    Estimates 1→k (split) and k→1 (merge) token events from alignment.
+    Higher values indicate more dramatic tokenization changes.
+
+    MSI = (splits + merges) / max(m, n) ∈ [0, 1]
+
+    Args:
+        original_tokens: Original token sequence(s).
+        corrupted_tokens: Corrupted token sequence(s).
+
+    Returns:
+        Merge-split index in [0, 1], or list for batches.
+    """
+    return _dispatch_metric(
+        original_tokens,
+        corrupted_tokens,
+        single=_single_msi,
+        batch=_batch_msi,
+        name="merge_split_index",
+    )
+
+
+# ---------------------------------------------------------------------------
+# MetricName Enum
+# ---------------------------------------------------------------------------
+
+
+class MetricName(str, Enum):
+    """Built-in metric names.
+
+    Use these instead of string literals to avoid typos and enable IDE completion.
+
+    Example:
+        >>> attack = Attack(Typogre(), metrics={MetricName.NED: normalized_edit_distance})
+        >>> # or get all defaults:
+        >>> attack = Attack(Typogre(), metrics=MetricName.defaults())
+    """
+
+    JSD = "jensen_shannon_divergence"
+    NED = "normalized_edit_distance"
+    SR = "subsequence_retention"
+    HD = "entropy_delta"
+    MSI = "merge_split_index"
+
+    @property
+    def func(self) -> "Callable[..., float | list[float]]":
+        """Get the metric function for this name."""
+        return _METRIC_FUNCTIONS[self]
+
+    @classmethod
+    def defaults(cls) -> dict[str, "Callable[..., float | list[float]]"]:
+        """Get all built-in metrics as a dictionary.
+
+        Returns:
+            Dictionary mapping metric names to functions.
+        """
+        return {m.value: m.func for m in cls}
+
+
+# Mapping from enum to function - populated after functions are defined
+_METRIC_FUNCTIONS: dict[MetricName, "Callable[..., float | list[float]]"] = {
+    MetricName.JSD: jensen_shannon_divergence,
+    MetricName.NED: normalized_edit_distance,
+    MetricName.SR: subsequence_retention,
+    MetricName.HD: entropy_delta,
+    MetricName.MSI: merge_split_index,
+}
+
+
+__all__ = [
+    "Metric",
+    "BatchMetric",
+    "MetricName",
+    "TokenBatch",
+    "TokenSequence",
+    "jensen_shannon_divergence",
+    "normalized_edit_distance",
+    "subsequence_retention",
+    "entropy_delta",
+    "merge_split_index",
+]

glitchlings/attack/metrics_dispatch.py

@@ -0,0 +1,70 @@
+"""Pure metric dispatch functions.
+
+This module contains pure functions for dispatching metric computations.
+It does not import Rust FFI or perform any IO - it operates on already-
+resolved metric functions.
+
+Pure guarantees:
+- No import side effects beyond stdlib
+- No Rust FFI loading
+- Deterministic dispatch logic
+
+The impure Rust metric loading lives in metrics.py.
+"""
+
+from __future__ import annotations
+
+from typing import Sequence, TypeGuard
+
+TokenSequence = Sequence[str]
+TokenBatch = Sequence[TokenSequence]
+
+
+def is_batch(tokens: TokenSequence | TokenBatch) -> TypeGuard[TokenBatch]:
+    """Determine if tokens represent a batch of sequences.
+
+    An empty list is treated as an empty batch (returning True) so that
+    ``metric([], [])`` returns ``[]`` rather than ``0.0``. This matches
+    the behavior of :meth:`Attack.run` when processing empty transcripts.
+
+    Args:
+        tokens: Either a sequence of token strings or a batch of such sequences.
+
+    Returns:
+        True if tokens is a batch (list of lists), False if a single sequence.
+    """
+    if not tokens:
+        return True  # Empty list is an empty batch
+
+    first = tokens[0]
+    return isinstance(first, Sequence) and not isinstance(first, (str, bytes))
+
+
+def validate_batch_consistency(
+    original: TokenSequence | TokenBatch,
+    corrupted: TokenSequence | TokenBatch,
+    metric_name: str,
+) -> None:
+    """Validate that both inputs are consistently batched or single.
+
+    Args:
+        original: Original token sequence or batch.
+        corrupted: Corrupted token sequence or batch.
+        metric_name: Name of the metric (for error messages).
+
+    Raises:
+        TypeError: If one input is batched and the other isn't.
+    """
+    original_is_batch = is_batch(original)
+    corrupted_is_batch = is_batch(corrupted)
+
+    if original_is_batch != corrupted_is_batch:
+        raise TypeError(f"{metric_name} expects either both batch inputs or both single sequences")
+
+
+__all__ = [
+    "TokenBatch",
+    "TokenSequence",
+    "is_batch",
+    "validate_batch_consistency",
+]

glitchlings/attack/tokenization.py

@@ -0,0 +1,227 @@
+from __future__ import annotations
+
+import importlib.util
+import zlib
+from typing import Any, Protocol, Sequence
+
+DEFAULT_TIKTOKEN_ENCODINGS = ("o200k_base", "cl100k_base")
+
+
+class Tokenizer(Protocol):
+    def encode(self, text: str) -> tuple[list[str], list[int]]: ...
+
+    def decode(self, tokens: Sequence[str]) -> str: ...
+
+
+class WhitespaceTokenizer:
+    def encode(self, text: str) -> tuple[list[str], list[int]]:
+        tokens = text.split()
+        # Synthetic IDs based on adler32 hash for stability
+        ids = [zlib.adler32(t.encode("utf-8")) & 0xFFFFFFFF for t in tokens]
+        return tokens, ids
+
+    def decode(self, tokens: Sequence[str]) -> str:
+        return " ".join(tokens)
+
+    def encode_batch(self, texts: Sequence[str]) -> list[tuple[list[str], list[int]]]:
+        return [self.encode(text) for text in texts]
+
+
+class TiktokenTokenizer:
+    def __init__(self, model_name: str):
+        import tiktoken
+
+        self.name = model_name
+        try:
+            self.enc = tiktoken.get_encoding(model_name)
+        except ValueError:
+            self.enc = tiktoken.encoding_for_model(model_name)
+
+    def encode(self, text: str) -> tuple[list[str], list[int]]:
+        ids = self.enc.encode(text)
+        tokens = [
+            self.enc.decode_single_token_bytes(i).decode("utf-8", errors="replace") for i in ids
+        ]
+        return tokens, ids
+
+    def decode(self, tokens: Sequence[str], sep: str = "") -> str:
+        return sep.join(tokens)
+
+    def encode_batch(self, texts: Sequence[str]) -> list[tuple[list[str], list[int]]]:
+        id_batches = [list(batch) for batch in self.enc.encode_batch(list(texts))]
+        token_batches: list[list[str]] = []
+        for ids in id_batches:
+            token_batches.append(
+                [
+                    self.enc.decode_single_token_bytes(i).decode("utf-8", errors="replace")
+                    for i in ids
+                ]
+            )
+        return list(zip(token_batches, id_batches))
+
+
+class HuggingFaceTokenizerWrapper:
+    def __init__(self, tokenizer_obj: Any, *, unknown_token: str = "[UNK]"):
+        self.tokenizer = tokenizer_obj
+        self.unknown_token = unknown_token
+
+    def encode(self, text: str) -> tuple[list[str], list[int]]:
+        # tokenizers.Tokenizer.encode returns an Encoding object
+        encoding = self.tokenizer.encode(text)
+        return encoding.tokens, encoding.ids
+
+    def decode(self, tokens: Sequence[str]) -> str:
+        # Use the tokenizer's decode method to properly handle model-specific
+        # artifacts (e.g., "##" for WordPiece, "Ġ" for BPE).
+        # Convert tokens to IDs first, then decode.
+        try:
+            token_ids = [self.tokenizer.token_to_id(token) for token in tokens]
+            # Filter out None values (tokens not in vocabulary)
+            valid_ids = [tid for tid in token_ids if tid is not None]
+            if valid_ids:
+                result: str = self.tokenizer.decode(valid_ids)
+                return result
+        except (AttributeError, TypeError):
+            pass
+        # Fallback: decode each token individually to handle artifacts properly
+        decoded_tokens = []
+        for token in tokens:
+            token_id = None
+            try:
+                token_id = self.tokenizer.token_to_id(token)
+            except (AttributeError, TypeError):
+                pass
+            if token_id is None:
+                decoded_tokens.append(self.unknown_token)
+            else:
+                # Decode the single token ID to properly handle artifacts
+                try:
+                    decoded = self.tokenizer.decode([token_id])
+                    decoded_tokens.append(decoded)
+                except (AttributeError, TypeError):
+                    # Last resort: strip common prefixes and use token as-is
+                    clean_token = token.lstrip("Ġ").lstrip("##").lstrip("▁")
+                    decoded_tokens.append(clean_token if clean_token else token)
+        return " ".join(decoded_tokens)
+
+    def encode_batch(self, texts: Sequence[str]) -> list[tuple[list[str], list[int]]]:
+        encodings = self.tokenizer.encode_batch(list(texts))
+        return [(encoding.tokens, encoding.ids) for encoding in encodings]
+
+
+def list_available_tokenizers() -> list[str]:
+    """List tokenizer names that can be resolved.
+
+    Returns a list of known tokenizer names including:
+    - Tiktoken encodings (if tiktoken is installed)
+    - A note about HuggingFace tokenizers (if tokenizers is installed)
+    - 'whitespace' (always available)
+
+    Returns:
+        List of available tokenizer names/descriptions.
+    """
+    available: list[str] = []
+
+    if importlib.util.find_spec("tiktoken"):
+        import tiktoken
+
+        # Add known tiktoken encodings
+        for encoding in DEFAULT_TIKTOKEN_ENCODINGS:
+            try:
+                tiktoken.get_encoding(encoding)
+                available.append(encoding)
+            except ValueError:
+                pass
+        # Add common model names
+        available.extend(["gpt-4", "gpt-4o", "gpt-3.5-turbo"])
+
+    if importlib.util.find_spec("tokenizers"):
+        available.append("<any HuggingFace tokenizer name>")
+
+    available.append("whitespace")
+    return available
+
+
+def resolve_tokenizer(tokenizer: str | Tokenizer | None) -> Tokenizer:
+    """Resolve a tokenizer specification to a Tokenizer instance.
+
+    Args:
+        tokenizer: One of:
+            - None: Use default tokenizer (tiktoken o200k_base, or whitespace)
+            - str: Tokenizer name (tiktoken encoding, model name, or HF tokenizer)
+            - Tokenizer: Pass through as-is
+
+    Returns:
+        A Tokenizer instance.
+
+    Raises:
+        ValueError: If string tokenizer cannot be resolved.
+    """
+    if tokenizer is None:
+        return _default_tokenizer()
+
+    if isinstance(tokenizer, str):
+        if importlib.util.find_spec("tiktoken"):
+            import tiktoken
+
+            try:
+                # Check if valid tiktoken encoding/model
+                try:
+                    tiktoken.get_encoding(tokenizer)
+                    return TiktokenTokenizer(tokenizer)
+                except ValueError:
+                    try:
+                        tiktoken.encoding_for_model(tokenizer)
+                        return TiktokenTokenizer(tokenizer)
+                    except (ValueError, KeyError):
+                        pass
+            except ImportError:
+                pass
+
+        if importlib.util.find_spec("tokenizers"):
+            from tokenizers import Tokenizer
+
+            try:
+                return HuggingFaceTokenizerWrapper(Tokenizer.from_pretrained(tokenizer))
+            except Exception:
+                pass
+
+        available = list_available_tokenizers()
+        raise ValueError(
+            f"Could not resolve tokenizer: {tokenizer!r}. Available: {', '.join(available)}"
+        )
+
+    # Check if it is a HuggingFace tokenizer object
+    if importlib.util.find_spec("tokenizers"):
+        from tokenizers import Tokenizer as HFTokenizer
+
+        if isinstance(tokenizer, HFTokenizer):
+            return HuggingFaceTokenizerWrapper(tokenizer)
+
+    return tokenizer
+
+
+def _default_tokenizer() -> Tokenizer:
+    """Select a modern, lightweight tokenizer with graceful fallbacks."""
+    if importlib.util.find_spec("tiktoken"):
+        import tiktoken
+
+        for encoding in DEFAULT_TIKTOKEN_ENCODINGS:
+            try:
+                tiktoken.get_encoding(encoding)
+                return TiktokenTokenizer(encoding)
+            except ValueError:
+                continue
+
+    return WhitespaceTokenizer()
+
+
+__all__ = [
+    "DEFAULT_TIKTOKEN_ENCODINGS",
+    "HuggingFaceTokenizerWrapper",
+    "TiktokenTokenizer",
+    "Tokenizer",
+    "WhitespaceTokenizer",
+    "list_available_tokenizers",
+    "resolve_tokenizer",
+]