glitchlings 1.0.0__cp313-cp313-win_amd64.whl

glitchlings/dlc/nemo.py

@@ -0,0 +1,283 @@
+"""NVIDIA NeMo DataDesigner plugin for Glitchlings text corruption.
+
+This module provides a DataDesigner column generator that applies Glitchlings
+transformations to text columns, enabling deterministic text corruption for
+model robustness testing and adversarial augmentation.
+
+The plugin integrates with NeMo's experimental plugin system, exposing
+Glitchlings as a discoverable column generator.
+
+Example:
+    >>> from glitchlings.dlc.nemo import GlitchlingColumnConfig
+    >>> from data_designer import DataDesignerConfigBuilder
+    >>> builder = DataDesignerConfigBuilder()
+    >>> builder.add_column(
+    ...     GlitchlingColumnConfig(
+    ...         name="corrupted_prompt",
+    ...         source_column="prompt",
+    ...         glitchlings=["Typogre(rate=0.02)", "Mim1c(rate=0.01)"],
+    ...         seed=404,
+    ...     )
+    ... )
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, Literal, Union
+
+from ..auggie import Auggie
+from ..conf.loaders import load_attack_config
+from ..util.adapters import coerce_gaggle
+from ..zoo.core import Gaggle, Glitchling
+
+if TYPE_CHECKING:  # pragma: no cover - typing only
+    import pandas as pd
+
+# Type alias for flexible glitchling specification
+GlitchlingSpec = Union[
+    Gaggle,
+    Auggie,
+    Glitchling,
+    str,
+    Sequence[Union[str, Glitchling]],
+    Path,
+]
+
+
+def _resolve_gaggle(
+    spec: GlitchlingSpec,
+    *,
+    seed: int,
+) -> Gaggle:
+    """Resolve a flexible glitchling specification into a Gaggle.
+
+    Args:
+        spec: One of:
+            - A pre-constructed ``Gaggle``
+            - An ``Auggie`` builder (which is a Gaggle subclass)
+            - A single ``Glitchling`` instance
+            - A string glitchling name (e.g., "typogre")
+            - A list of glitchling names or instances
+            - A ``Path`` to a YAML config file
+        seed: Seed for deterministic corruption.
+
+    Returns:
+        A configured Gaggle ready for corruption.
+
+    Raises:
+        TypeError: If the spec type is not recognized.
+        ValueError: If a config path doesn't exist or is invalid.
+    """
+    # Auggie is a subclass of Gaggle, so check it first
+    if isinstance(spec, Auggie):
+        return spec.clone(seed=seed)
+
+    if isinstance(spec, Gaggle):
+        return spec.clone(seed=seed)
+
+    # Path to YAML config
+    if isinstance(spec, Path):
+        config = load_attack_config(spec)
+        from ..conf.loaders import build_gaggle
+
+        return build_gaggle(config, seed_override=seed)
+
+    # String path (check if it looks like a file path)
+    if isinstance(spec, str) and (spec.endswith(".yaml") or spec.endswith(".yml")):
+        path = Path(spec)
+        if not path.exists():
+            raise FileNotFoundError(
+                f"Glitchling config file not found: {spec!r}. "
+                "If this was intended as a glitchling name, remove the .yaml/.yml extension."
+            )
+        config = load_attack_config(path)
+        from ..conf.loaders import build_gaggle
+
+        return build_gaggle(config, seed_override=seed)
+
+    # Single glitchling or string, or sequence
+    return coerce_gaggle(spec, seed=seed)
+
+
+def _apply_corruption(
+    series: "pd.Series[str]",
+    gaggle: Gaggle,
+) -> "pd.Series[str]":
+    """Apply gaggle corruption to a pandas Series.
+
+    Uses batch corruption for efficiency when possible.
+
+    Args:
+        series: Pandas Series of strings to corrupt.
+        gaggle: Configured Gaggle to apply.
+
+    Returns:
+        Series with corrupted strings.
+    """
+    # Use batch corruption for better performance via Rust pipeline
+    values = series.tolist()
+    corrupted = gaggle.corrupt_batch(values)
+    return series.__class__(corrupted, index=series.index, name=series.name)
+
+
+# ---------------------------------------------------------------------------
+# DataDesigner Integration Classes
+# ---------------------------------------------------------------------------
+# These classes follow the NeMo DataDesigner plugin interface.
+# They are defined conditionally to avoid hard dependency on data-designer.
+
+
+def _create_plugin_classes() -> tuple[type, type, Any] | None:
+    """Create plugin classes if data-designer is available.
+
+    Returns:
+        Tuple of (config_class, generator_class, plugin_object) or None.
+    """
+    try:
+        from data_designer.config.base import SingleColumnConfig
+        from data_designer.engine.column_generators.generators.base import (
+            ColumnGenerator,
+            GenerationStrategy,
+            GeneratorMetadata,
+        )
+        from data_designer.plugins import Plugin, PluginType
+    except ImportError:
+        return None
+
+    class GlitchlingColumnConfig(SingleColumnConfig):  # type: ignore[misc]
+        """Configuration for Glitchlings text corruption column generator.
+
+        Attributes:
+            name: Output column name.
+            column_type: Discriminator field for DataDesigner plugin discovery.
+            glitchlings: Glitchling specification. Can be:
+                - A string glitchling name: ``"typogre"``
+                - A spec with parameters: ``"Typogre(rate=0.02)"``
+                - A list of specs: ``["Typogre(rate=0.02)", "Mim1c(rate=0.01)"]``
+                - A path to YAML config: ``"configs/chaos.yaml"``
+            source_column: Column to corrupt. If None, corrupts the column
+                specified by ``name`` (in-place style).
+            seed: RNG seed for deterministic corruption. If None, uses
+                a default seed for reproducibility.
+        """
+
+        column_type: Literal["glitchlings"] = "glitchlings"
+        glitchlings: str | list[str] = "typogre"
+        source_column: str | None = None
+        seed: int | None = None
+
+    class GlitchlingColumnGenerator(ColumnGenerator):  # type: ignore[misc]
+        """Column generator that applies Glitchlings text corruption.
+
+        This generator corrupts text in the source column using the configured
+        glitchlings and writes the result to the output column.
+        """
+
+        config: GlitchlingColumnConfig
+
+        @staticmethod
+        def metadata() -> GeneratorMetadata:
+            """Return metadata describing this generator."""
+            return GeneratorMetadata(
+                name="glitchlings",
+                description=(
+                    "Apply deterministic, linguistically-principled text corruption "
+                    "via Glitchlings for model robustness testing and adversarial augmentation."
+                ),
+                generation_strategy=GenerationStrategy.FULL_COLUMN,
+                required_resources=None,
+            )
+
+        def generate(self, data: "pd.DataFrame") -> "pd.DataFrame":
+            """Generate corrupted text column.
+
+            Args:
+                data: Input DataFrame.
+
+            Returns:
+                DataFrame with the corrupted column added/updated.
+            """
+            source = self.config.source_column or self.config.name
+            seed = self.config.seed if self.config.seed is not None else 151
+
+            # Resolve glitchlings specification
+            spec: GlitchlingSpec = self.config.glitchlings
+
+            gaggle = _resolve_gaggle(spec, seed=seed)
+
+            # Apply corruption
+            data[self.config.name] = _apply_corruption(data[source], gaggle)
+            return data
+
+    plugin = Plugin(
+        task_cls=GlitchlingColumnGenerator,
+        config_cls=GlitchlingColumnConfig,
+        plugin_type=PluginType.COLUMN_GENERATOR,
+        emoji="👾",
+    )
+
+    return GlitchlingColumnConfig, GlitchlingColumnGenerator, plugin
+
+
+# Try to create the plugin classes
+_plugin_result = _create_plugin_classes()
+
+if _plugin_result is not None:
+    GlitchlingColumnConfig, GlitchlingColumnGenerator, plugin = _plugin_result
+else:
+    # Provide stub classes for documentation and type checking
+    GlitchlingColumnConfig = None  # type: ignore[assignment]
+    GlitchlingColumnGenerator = None  # type: ignore[assignment]
+    plugin = None
+
+
+# ---------------------------------------------------------------------------
+# Standalone Functions (usable without DataDesigner)
+# ---------------------------------------------------------------------------
+
+
+def corrupt_dataframe(
+    df: "pd.DataFrame",
+    glitchlings: GlitchlingSpec,
+    *,
+    column: str,
+    output_column: str | None = None,
+    seed: int = 151,
+) -> "pd.DataFrame":
+    """Corrupt a DataFrame column using Glitchlings.
+
+    This function provides DataFrame corruption without requiring the full
+    DataDesigner plugin infrastructure.
+
+    Args:
+        df: Input DataFrame.
+        glitchlings: Glitchling specification (see ``GlitchlingSpec``).
+        column: Source column to corrupt.
+        output_column: Output column name. If None, overwrites source column.
+        seed: RNG seed for deterministic corruption.
+
+    Returns:
+        DataFrame with corrupted column.
+
+    Example:
+        >>> import pandas as pd
+        >>> from glitchlings.dlc.nemo import corrupt_dataframe
+        >>> df = pd.DataFrame({"text": ["Hello world", "Test input"]})
+        >>> result = corrupt_dataframe(df, "typogre", column="text", seed=42)
+    """
+    gaggle = _resolve_gaggle(glitchlings, seed=seed)
+    target = output_column if output_column is not None else column
+    df = df.copy()
+    df[target] = _apply_corruption(df[column], gaggle)
+    return df
+
+
+__all__ = [
+    "GlitchlingColumnConfig",
+    "GlitchlingColumnGenerator",
+    "GlitchlingSpec",
+    "corrupt_dataframe",
+    "plugin",
+]

glitchlings/dlc/prime.py

@@ -0,0 +1,215 @@
+"""Integration helpers for the optional verifiers prime DLC."""
+
+from __future__ import annotations
+
+from collections.abc import Iterable, Sequence
+from typing import Any, Callable, Protocol, cast
+
+from ..compat.loaders import require_datasets, require_jellyfish, require_verifiers
+from ..util.adapters import coerce_gaggle
+from ..zoo import Gaggle, Glitchling, Mim1c, Typogre  # noqa: F401
+from ._shared import resolve_columns as _resolve_columns_shared
+
+
+class VerifierEnvironment(Protocol):
+    """Minimal interface for verifiers environments."""
+
+    dataset: Any
+
+
+class VerifierSingleTurnEnv(Protocol):
+    """Minimal interface for single-turn verifier environments."""
+
+    dataset: Any
+    rubric: Any
+
+
+vf = require_verifiers("verifiers is not installed; install glitchlings[prime]")
+_jellyfish = require_jellyfish("jellyfish is not installed; install glitchlings[prime]")
+levenshtein_distance = _jellyfish.levenshtein_distance
+
+
+def _resolve_environment(env: str | VerifierEnvironment) -> VerifierEnvironment:
+    """Return a fully-instantiated verifier environment."""
+    if isinstance(env, str):
+        env = vf.load_environment(env)
+
+    if not isinstance(env, cast(type[Any], vf.Environment)):
+        raise TypeError("Invalid environment type")
+
+    return cast(VerifierEnvironment, env)
+
+
+def _resolve_columns(dataset: Any, columns: Sequence[str] | None) -> list[str]:
+    """Identify which dataset columns should be corrupted."""
+    return _resolve_columns_shared(dataset, columns)
+
+
+def load_environment(
+    env: str | VerifierEnvironment,
+    glitchlings: Iterable[str | Glitchling] | Glitchling | str | Gaggle | None = None,
+    *,
+    seed: int = 151,
+    columns: Sequence[str] | None = None,
+) -> VerifierEnvironment:
+    """Load an environment and optionally corrupt it with glitchlings."""
+    environment = _resolve_environment(env)
+
+    if glitchlings is None:
+        return environment
+
+    gaggle = coerce_gaggle(glitchlings, seed=seed)
+
+    dataset = environment.dataset
+    corrupt_columns = _resolve_columns(dataset, columns)
+    environment.dataset = gaggle.corrupt_dataset(dataset, corrupt_columns)
+    return environment
+
+
+def _as_gaggle(
+    glitchlings: Iterable[str | Glitchling] | Glitchling | str | Gaggle,
+    *,
+    seed: int,
+) -> Gaggle:
+    """Coerce any supported glitchling specification into a :class:`Gaggle`."""
+    return coerce_gaggle(glitchlings, seed=seed)
+
+
+def _extract_completion_text(completion: Any) -> str:
+    """Normalize a completion payload into a plain string."""
+    if isinstance(completion, str):
+        return completion
+
+    if isinstance(completion, list) and completion:
+        first = completion[0]
+        if isinstance(first, dict) and "content" in first:
+            return str(first["content"])
+        return str(first)
+
+    return str(completion)
+
+
+def normalized_edit_distance(
+    _: Any,
+    completion: Any,
+    answer: str,
+) -> float:
+    """Return ``1 - (distance / max_len)`` using Levenshtein distance."""
+    completion_text = _extract_completion_text(completion)
+    target = answer or ""
+    denominator = max(len(completion_text), len(target), 1)
+    distance = cast(int, levenshtein_distance(completion_text, target))
+    score = 1.0 - (distance / denominator)
+    return max(0.0, min(1.0, score))
+
+
+symmetric_levenshtein_similarity = normalized_edit_distance
+
+DEFAULT_CLEANUP_INSTRUCTIONS = (
+    "You are a meticulous copy editor. Restore the provided text to its original form."
+)
+
+
+def echo_chamber(
+    dataset_id: str,
+    column: str,
+    glitchlings: Iterable[str | Glitchling] | Glitchling | str | Gaggle,
+    *,
+    seed: int = 151,
+    instructions: str = DEFAULT_CLEANUP_INSTRUCTIONS,
+    reward_function: Callable[..., float] | None = None,
+    split: str | None = None,
+    **load_dataset_kwargs: Any,
+) -> VerifierSingleTurnEnv:
+    """Create an Echo Chamber Prime environment from a Hugging Face dataset column.
+
+    Args:
+        dataset_id: Identifier of the Hugging Face dataset to load.
+        column: Name of the column whose text should be glitched.
+        glitchlings: Glitchling specifiers that will corrupt the prompts.
+        seed: RNG seed forwarded to :func:`glitchlings.util.adapters.coerce_gaggle`.
+        instructions: System instructions supplied to the environment prompts.
+        reward_function: Optional callable used to score completions. Defaults to
+            :func:`symmetric_levenshtein_similarity` when omitted.
+        split: Optional dataset split to load.
+        **load_dataset_kwargs: Extra keyword arguments forwarded to
+            :func:`datasets.load_dataset`.
+
+    """
+    datasets_module = require_datasets("datasets is required to build an echo chamber")
+    load_dataset = getattr(datasets_module, "load_dataset", None)
+    if load_dataset is None:  # pragma: no cover - defensive
+        message = "datasets is required to build an echo chamber"
+        raise ModuleNotFoundError(message)
+
+    dataset_dict_cls = getattr(datasets_module, "DatasetDict", dict)
+
+    hf_dataset: Any
+    if split is None:
+        hf_dataset = load_dataset(dataset_id, **load_dataset_kwargs)
+        if isinstance(hf_dataset, dataset_dict_cls):
+            try:
+                hf_dataset = next(iter(hf_dataset.values()))
+            except StopIteration as exc:  # pragma: no cover - defensive
+                raise ValueError("The specified dataset does not contain any splits") from exc
+    else:
+        hf_dataset = load_dataset(dataset_id, split=split, **load_dataset_kwargs)
+
+    if isinstance(hf_dataset, dataset_dict_cls):
+        raise ValueError("Specify which split to use when the dataset loads as a DatasetDict.")
+
+    filtered_dataset = hf_dataset.filter(
+        lambda row: row.get(column) is not None,
+        load_from_cache_file=False,
+    )
+
+    source_column_names = list(filtered_dataset.column_names)
+
+    def _build_prompt(row: dict[str, Any]) -> dict[str, Any]:
+        text = str(row[column])
+        prompt = [
+            {"role": "system", "content": instructions},
+            {"role": "user", "content": f"Corrupted text:\n{text}"},
+        ]
+        return {"prompt": prompt, "answer": text}
+
+    base_dataset = filtered_dataset.map(
+        _build_prompt,
+        remove_columns=source_column_names,
+        load_from_cache_file=False,
+    )
+
+    try:
+        dataset_length = len(base_dataset)
+    except TypeError:
+        preview_rows: list[dict[str, Any]]
+        take_fn = getattr(base_dataset, "take", None)
+        if callable(take_fn):
+            preview_rows = list(take_fn(1))
+        else:
+            iterator = iter(base_dataset)
+            try:
+                first_row = next(iterator)
+            except StopIteration:
+                preview_rows = []
+            else:
+                preview_rows = [first_row]
+        if not preview_rows:
+            raise ValueError(
+                f"Column '{column}' did not yield any textual entries in dataset '{dataset_id}'."
+            )
+    else:
+        if dataset_length == 0:
+            raise ValueError(
+                f"Column '{column}' did not yield any textual entries in dataset '{dataset_id}'."
+            )
+
+    gaggle = _as_gaggle(glitchlings, seed=seed)
+    glitched_dataset = gaggle.corrupt_dataset(base_dataset, ["prompt"])
+
+    rubric_func = reward_function or normalized_edit_distance
+    rubric = vf.Rubric(funcs=[rubric_func], weights=[1.0])
+    return cast(
+        VerifierSingleTurnEnv,
+        vf.SingleTurnEnv(dataset=glitched_dataset, rubric=rubric),
+    )

glitchlings/dlc/pytorch.py

@@ -0,0 +1,98 @@
+"""Integration helpers for PyTorch data loaders."""
+
+from __future__ import annotations
+
+from collections.abc import Iterable, Iterator, Sequence
+from typing import Any, cast
+
+from ..util.adapters import coerce_gaggle
+from ..zoo import Gaggle, Glitchling
+from ._shared import corrupt_batch, infer_batch_targets, normalize_column_spec
+
+
+class _GlitchedDataLoader(Iterable[Any]):
+    """Wrapper that applies glitchlings lazily to each batch from a data loader."""
+
+    def __init__(
+        self,
+        dataloader: Any,
+        gaggle: Gaggle,
+        *,
+        columns: list[str | int] | None,
+    ) -> None:
+        self._dataloader = dataloader
+        self._gaggle = gaggle
+        self._explicit_columns = columns
+        self._inferred_columns: list[str | int] | None | _Sentinel = _UNINITIALISED
+
+    def __iter__(self) -> Iterator[Any]:
+        # Reset all glitchling RNGs before each fresh pass for determinism.
+        self._gaggle.sort_glitchlings()
+        for batch in self._dataloader:
+            targets = self._resolve_columns(batch)
+            yield corrupt_batch(batch, targets, self._gaggle)
+
+    def __len__(self) -> int:
+        return len(self._dataloader)
+
+    def __getattr__(self, attribute: str) -> Any:
+        return getattr(self._dataloader, attribute)
+
+    def _resolve_columns(self, batch: Any) -> list[str | int] | None:
+        if self._explicit_columns is not None:
+            return self._explicit_columns
+
+        if self._inferred_columns is _UNINITIALISED:
+            self._inferred_columns = infer_batch_targets(batch)
+
+        return cast(list[str | int] | None, self._inferred_columns)
+
+
+class _Sentinel:
+    """Sentinel type for deferred column inference."""
+
+
+_UNINITIALISED = _Sentinel()
+
+
+def GlitchedDataLoader(
+    dataloader: Any,
+    glitchlings: Iterable[str | Glitchling] | Glitchling | str | Gaggle,
+    *,
+    columns: str | int | Sequence[str | int] | None = None,
+    seed: int = 151,
+) -> _GlitchedDataLoader:
+    """Return a lazily glitched view of a PyTorch DataLoader's batches.
+
+    This function wraps a PyTorch DataLoader to apply glitchlings to specified
+    columns (or auto-inferred text columns) in each batch as it's yielded.
+
+    Args:
+        dataloader: The PyTorch DataLoader to wrap.
+        glitchlings: A glitchling, gaggle, or specification of glitchlings to apply.
+        columns: Column name(s) or index/indices to corrupt. Can be:
+                 - A single string column name (for dict-like batches)
+                 - A single integer index (for sequence-like batches)
+                 - A sequence of column names or indices
+                 - None to auto-infer text columns (default)
+        seed: RNG seed for deterministic corruption (default: 151).
+
+    Returns:
+        A wrapped dataloader that yields corrupted batches.
+
+    Example:
+        >>> from torch.utils.data import DataLoader
+        >>> from glitchlings.dlc.pytorch import GlitchedDataLoader
+        >>> dataset = [{"text": "hello", "label": 0}]
+        >>> loader = DataLoader(dataset)
+        >>> glitched = GlitchedDataLoader(loader, "typogre", columns="text")
+        >>> for batch in glitched:
+        ...     print(batch)
+        {'text': 'helo', 'label': 0}
+    """
+    gaggle = coerce_gaggle(glitchlings, seed=seed)
+    normalized = normalize_column_spec(columns)
+    return _GlitchedDataLoader(dataloader, gaggle, columns=normalized)
+
+
+__all__ = ["GlitchedDataLoader"]