bead 0.1.0__py3-none-any.whl

bead/resources/template_generation.py

@@ -0,0 +1,349 @@
+"""Abstract base class for mapping external frame inventories to Templates.
+
+This module provides language-agnostic base classes for generating Template
+objects from external linguistic frame inventories (e.g., VerbNet, FrameNet,
+PropBank, valency lexicons).
+"""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import Any
+
+from bead.resources.constraints import Constraint
+from bead.resources.template import Slot, Template
+
+
+class FrameToTemplateMapper(ABC):
+    """Abstract base class for mapping frame inventories to Templates.
+
+    This class provides a framework for generating Template objects from
+    external linguistic frame data. Subclasses implement language- and
+    resource-specific mapping logic.
+
+    Examples
+    --------
+    Implementing a VerbNet mapper:
+    >>> class VerbNetMapper(FrameToTemplateMapper):
+    ...     def generate_from_frame(self, verb_lemma, frame_data):
+    ...         slots = self.map_frame_to_slots(frame_data)
+    ...         constraints = self.generate_constraints(frame_data, slots)
+    ...         return Template(
+    ...             name=f"{verb_lemma}_{frame_data['id']}",
+    ...             template_string=frame_data['template_string'],
+    ...             slots=slots,
+    ...             constraints=constraints
+    ...         )
+    ...
+    ...     def map_frame_to_slots(self, frame_data):
+    ...         # Extract slots from VerbNet syntax
+    ...         return {}
+    ...
+    ...     def generate_constraints(self, frame_data, slots):
+    ...         # Generate constraints from VerbNet restrictions
+    ...         return []
+    """
+
+    @abstractmethod
+    def generate_from_frame(
+        self,
+        *args: Any,
+        **kwargs: Any,
+    ) -> Template | list[Template]:
+        """Generate Template(s) from a frame specification.
+
+        This is the main entry point for template generation. Subclasses
+        implement the specific logic for their frame inventory.
+
+        Parameters
+        ----------
+        *args : Any
+            Positional arguments (frame data, identifiers, etc.).
+        **kwargs : Any
+            Keyword arguments (configuration options, etc.).
+
+        Returns
+        -------
+        Template | list[Template]
+            Generated template(s). May return multiple templates if the
+            frame has multiple realizations (e.g., different complementizer
+            types, alternations).
+
+        Examples
+        --------
+        VerbNet implementation:
+        >>> mapper.generate_from_frame(
+        ...     verb_lemma="think",
+        ...     verbnet_class="29.9",
+        ...     frame_data={"primary": "NP V that S"}
+        ... )  # doctest: +SKIP
+        """
+        ...
+
+    @abstractmethod
+    def map_frame_to_slots(
+        self,
+        frame_data: Any,
+    ) -> dict[str, Slot]:
+        """Map frame elements to Template slots.
+
+        Converts frame-specific element descriptions into Slot objects
+        with appropriate constraints.
+
+        Parameters
+        ----------
+        frame_data : Any
+            Frame specification from the external inventory.
+            Type depends on the specific resource (dict, object, etc.).
+
+        Returns
+        -------
+        dict[str, Slot]
+            Slots keyed by slot name.
+
+        Examples
+        --------
+        Mapping VerbNet syntax to slots:
+        >>> slots = mapper.map_frame_to_slots({
+        ...     "syntax": [
+        ...         ("NP", "Agent"),
+        ...         ("V", None),
+        ...         ("NP", "Theme")
+        ...     ]
+        ... })  # doctest: +SKIP
+        >>> "subject" in slots
+        True
+        """
+        ...
+
+    @abstractmethod
+    def generate_constraints(
+        self,
+        frame_data: Any,
+        slots: dict[str, Slot],
+    ) -> list[Constraint]:
+        """Generate multi-slot constraints from frame specifications.
+
+        Converts frame-specific restrictions into DSL Constraint objects
+        that enforce relationships between slots.
+
+        Parameters
+        ----------
+        frame_data : Any
+            Frame specification from the external inventory.
+        slots : dict[str, Slot]
+            Slots that have been created for this frame.
+
+        Returns
+        -------
+        list[Constraint]
+            Multi-slot constraints for the template.
+
+        Examples
+        --------
+        Generating constraints from VerbNet restrictions:
+        >>> constraints = mapper.generate_constraints(
+        ...     frame_data={"restrictions": [...]},
+        ...     slots={"subject": ..., "verb": ...}
+        ... )  # doctest: +SKIP
+        """
+        ...
+
+    def create_template_name(
+        self,
+        *identifiers: str,
+        separator: str = "_",
+    ) -> str:
+        """Create a unique template name from identifiers.
+
+        Utility method for generating consistent template names.
+        Sanitizes identifiers by replacing spaces, dots, and hyphens.
+
+        Parameters
+        ----------
+        *identifiers : str
+            Components to include in the name (e.g., verb, class, frame).
+        separator : str
+            Separator between components (default: "_").
+
+        Returns
+        -------
+        str
+            Sanitized template name.
+
+        Examples
+        --------
+        >>> mapper = ConcreteMapper()
+        >>> mapper.create_template_name("think", "29.9", "that-clause")
+        'think_29_9_that_clause'
+        """
+        # sanitize each identifier
+        sanitized: list[str] = []
+        for identifier in identifiers:
+            safe: str = (
+                identifier.replace(" ", separator)
+                .replace(".", separator)
+                .replace("-", separator)
+            )
+            sanitized.append(safe)
+
+        return separator.join(sanitized)
+
+    def create_template_metadata(
+        self,
+        frame_data: dict[str, Any],
+        **additional_metadata: Any,
+    ) -> dict[str, Any]:
+        """Create metadata dictionary for template.
+
+        Utility method for extracting and organizing frame metadata.
+        Subclasses can override to add resource-specific metadata.
+
+        Parameters
+        ----------
+        frame_data : dict[str, Any]
+            Frame specification from the external inventory.
+        **additional_metadata : Any
+            Additional metadata to include.
+
+        Returns
+        -------
+        dict[str, Any]
+            Metadata dictionary for Template.metadata field.
+
+        Examples
+        --------
+        >>> mapper = ConcreteMapper()
+        >>> metadata = mapper.create_template_metadata(
+        ...     frame_data={"id": "29.9-1", "examples": [...]},
+        ...     verb_lemma="think"
+        ... )  # doctest: +SKIP
+        """
+        metadata: dict[str, Any] = {}
+
+        # add frame data
+        metadata.update(frame_data)
+
+        # add additional metadata
+        metadata.update(additional_metadata)
+
+        return metadata
+
+
+class MultiFrameMapper(FrameToTemplateMapper):
+    """Mapper that generates multiple template variants from a single frame.
+
+    Some frame specifications support multiple realizations (e.g., different
+    complementizer types, voice alternations). This class provides a framework
+    for generating all variants.
+
+    Examples
+    --------
+    >>> class ClausalMapper(MultiFrameMapper):
+    ...     def get_frame_variants(self, frame_data):
+    ...         # Return list of variant specifications
+    ...         return [
+    ...             {"comp": "that", "mood": "declarative"},
+    ...             {"comp": "whether", "mood": "interrogative"},
+    ...         ]
+    ...
+    ...     def generate_from_frame(self, verb, frame_data):
+    ...         variants = self.get_frame_variants(frame_data)
+    ...         return [self._generate_variant(verb, v) for v in variants]
+    ...
+    ...     def map_frame_to_slots(self, frame_data):
+    ...         return {}
+    ...
+    ...     def generate_constraints(self, frame_data, slots):
+    ...         return []
+    """
+
+    @abstractmethod
+    def get_frame_variants(
+        self,
+        frame_data: Any,
+    ) -> list[Any]:
+        """Extract all variants from frame specification.
+
+        Parameters
+        ----------
+        frame_data : Any
+            Frame specification from the external inventory.
+
+        Returns
+        -------
+        list[Any]
+            List of variant specifications, each representing one possible
+            realization of the frame.
+
+        Examples
+        --------
+        >>> variants = mapper.get_frame_variants({
+        ...     "complementizers": ["that", "whether", "if"]
+        ... })  # doctest: +SKIP
+        >>> len(variants)
+        3
+        """
+        ...
+
+    def generate_from_frame(
+        self,
+        *args: Any,
+        **kwargs: Any,
+    ) -> list[Template]:
+        """Generate templates for all frame variants.
+
+        Default implementation calls get_frame_variants() and generates
+        a template for each variant. Subclasses can override for custom logic.
+
+        Parameters
+        ----------
+        *args : Any
+            Positional arguments passed to variant generation.
+        **kwargs : Any
+            Keyword arguments passed to variant generation.
+
+        Returns
+        -------
+        list[Template]
+            Templates for all variants.
+        """
+        # extract frame_data from kwargs
+        frame_data = kwargs.get("frame_data")
+        if frame_data is None:
+            raise ValueError("frame_data must be provided in kwargs")
+
+        variants = self.get_frame_variants(frame_data)
+
+        templates: list[Template] = []
+        for variant in variants:
+            # create a modified kwargs with variant info
+            variant_kwargs = kwargs.copy()
+            variant_kwargs["variant_data"] = variant
+
+            template: Template = self._generate_variant(*args, **variant_kwargs)
+            templates.append(template)
+
+        return templates
+
+    @abstractmethod
+    def _generate_variant(
+        self,
+        *args: Any,
+        **kwargs: Any,
+    ) -> Template:
+        """Generate template for a single variant.
+
+        Parameters
+        ----------
+        *args : Any
+            Positional arguments.
+        **kwargs : Any
+            Keyword arguments, including variant_data.
+
+        Returns
+        -------
+        Template
+            Template for this variant.
+        """
+        ...

bead/simulation/__init__.py

@@ -0,0 +1,29 @@
+"""Simulation framework for generating synthetic human judgments.
+
+Provides annotators, noise models, and strategies for testing active
+learning pipelines without real human data.
+"""
+
+from bead.simulation.annotators.base import SimulatedAnnotator
+from bead.simulation.annotators.lm_based import LMBasedAnnotator
+from bead.simulation.noise_models.base import NoiseModel
+from bead.simulation.noise_models.temperature import TemperatureNoiseModel
+from bead.simulation.runner import SimulationRunner
+from bead.simulation.strategies.base import SimulationStrategy
+from bead.simulation.strategies.binary import BinaryStrategy
+from bead.simulation.strategies.categorical import CategoricalStrategy
+from bead.simulation.strategies.forced_choice import ForcedChoiceStrategy
+from bead.simulation.strategies.ordinal_scale import OrdinalScaleStrategy
+
+__all__ = [
+    "SimulatedAnnotator",
+    "LMBasedAnnotator",
+    "NoiseModel",
+    "TemperatureNoiseModel",
+    "SimulationRunner",
+    "SimulationStrategy",
+    "BinaryStrategy",
+    "CategoricalStrategy",
+    "ForcedChoiceStrategy",
+    "OrdinalScaleStrategy",
+]

bead/simulation/annotators/__init__.py

@@ -0,0 +1,15 @@
+"""Simulated annotators for generating synthetic judgments."""
+
+from bead.simulation.annotators.base import SimulatedAnnotator
+from bead.simulation.annotators.distance_based import DistanceBasedAnnotator
+from bead.simulation.annotators.lm_based import LMBasedAnnotator
+from bead.simulation.annotators.oracle import OracleAnnotator
+from bead.simulation.annotators.random import RandomAnnotator
+
+__all__ = [
+    "SimulatedAnnotator",
+    "DistanceBasedAnnotator",
+    "LMBasedAnnotator",
+    "OracleAnnotator",
+    "RandomAnnotator",
+]

bead/simulation/annotators/base.py

@@ -0,0 +1,175 @@
+"""Base class for simulated annotators."""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import TYPE_CHECKING
+
+import numpy as np
+
+if TYPE_CHECKING:
+    from bead.config.simulation import SimulatedAnnotatorConfig
+    from bead.items.item import Item
+    from bead.items.item_template import ItemTemplate
+    from bead.simulation.noise_models.base import NoiseModel
+    from bead.simulation.strategies.base import SimulationStrategy
+
+
+class SimulatedAnnotator(ABC):
+    """Abstract base for simulated annotators.
+
+    An annotator combines:
+    - Task-specific strategy (how to respond to each task type)
+    - Noise model (how to add human-like variability)
+    - Configuration (model output keys, random seed, etc.)
+
+    The annotator orchestrates the simulation process and provides
+    a unified interface for generating judgments.
+
+    Parameters
+    ----------
+    config
+        Configuration for annotator.
+    random_state
+        Random seed (overrides config if provided).
+    """
+
+    def __init__(
+        self, config: SimulatedAnnotatorConfig, random_state: int | None = None
+    ) -> None:
+        self.config = config
+        self.random_state = random_state or config.random_state
+        self.rng = np.random.RandomState(self.random_state)
+
+        # will be set by subclasses
+        self.strategies: dict[str, SimulationStrategy] = {}
+        self.noise_model: NoiseModel | None = None
+
+    @classmethod
+    def from_config(cls, config: SimulatedAnnotatorConfig) -> SimulatedAnnotator:
+        """Create annotator from configuration.
+
+        Parameters
+        ----------
+        config : SimulatedAnnotatorConfig
+            Configuration specifying annotator type and parameters.
+
+        Returns
+        -------
+        SimulatedAnnotator
+            Configured annotator instance.
+
+        Raises
+        ------
+        ValueError
+            If strategy is unknown.
+
+        Examples
+        --------
+        >>> from bead.config.simulation import SimulatedAnnotatorConfig
+        >>> config = SimulatedAnnotatorConfig(strategy="lm_score")
+        >>> annotator = SimulatedAnnotator.from_config(config)
+        """
+        # import here to avoid circular dependency
+        from bead.simulation.annotators.distance_based import (  # noqa: PLC0415
+            DistanceBasedAnnotator,
+        )
+        from bead.simulation.annotators.lm_based import (  # noqa: PLC0415
+            LMBasedAnnotator,
+        )
+        from bead.simulation.annotators.oracle import OracleAnnotator  # noqa: PLC0415
+        from bead.simulation.annotators.random import RandomAnnotator  # noqa: PLC0415
+
+        if config.strategy == "lm_score":
+            return LMBasedAnnotator(config)
+        elif config.strategy == "random":
+            return RandomAnnotator(config)
+        elif config.strategy == "oracle":
+            return OracleAnnotator(config)
+        elif config.strategy == "distance":
+            return DistanceBasedAnnotator(config)
+        else:
+            msg = f"Unknown strategy: {config.strategy}"
+            raise ValueError(msg)
+
+    @abstractmethod
+    def annotate(
+        self, item: Item, item_template: ItemTemplate
+    ) -> str | int | float | list[str]:
+        """Generate annotation for single item.
+
+        Parameters
+        ----------
+        item : Item
+            Item to annotate.
+        item_template : ItemTemplate
+            Template defining task structure.
+
+        Returns
+        -------
+        str | int | float | list[str]
+            Annotation (format depends on task type).
+        """
+
+    def annotate_batch(
+        self,
+        items: list[Item],
+        item_templates: list[ItemTemplate] | ItemTemplate,
+    ) -> dict[str, str | int | float | list[str]]:
+        """Generate annotations for batch of items.
+
+        Parameters
+        ----------
+        items : list[Item]
+            Items to annotate.
+        item_templates : list[ItemTemplate] | ItemTemplate
+            Templates (one per item or single template for all).
+
+        Returns
+        -------
+        dict[str, str | int | float | list[str]]
+            Mapping from item ID to annotation.
+
+        Examples
+        --------
+        >>> annotations = annotator.annotate_batch(items, template)
+        >>> annotations[str(items[0].id)]
+        'option_a'
+        """
+        # handle single template
+        templates_list: list[ItemTemplate]
+        if not isinstance(item_templates, list):
+            templates_list = [item_templates] * len(items)
+        else:
+            templates_list = item_templates
+
+        # annotate each item
+        annotations: dict[str, str | int | float | list[str]] = {}
+        for item, template in zip(items, templates_list, strict=True):
+            annotation = self.annotate(item, template)
+            annotations[str(item.id)] = annotation
+
+        return annotations
+
+    def get_strategy(self, task_type: str) -> SimulationStrategy:
+        """Get strategy for task type.
+
+        Parameters
+        ----------
+        task_type : str
+            Task type (e.g., "forced_choice").
+
+        Returns
+        -------
+        SimulationStrategy
+            Strategy for this task type.
+
+        Raises
+        ------
+        ValueError
+            If task type not supported.
+        """
+        if task_type not in self.strategies:
+            msg = f"Task type '{task_type}' not supported by {self.__class__.__name__}"
+            raise ValueError(msg)
+        return self.strategies[task_type]

bead/simulation/annotators/distance_based.py

@@ -0,0 +1,135 @@
+"""Distance-based annotator using embeddings."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from bead.simulation.annotators.base import SimulatedAnnotator
+from bead.simulation.strategies.binary import BinaryStrategy
+from bead.simulation.strategies.categorical import CategoricalStrategy
+from bead.simulation.strategies.cloze import ClozeStrategy
+from bead.simulation.strategies.forced_choice import ForcedChoiceStrategy
+from bead.simulation.strategies.free_text import FreeTextStrategy
+from bead.simulation.strategies.magnitude import MagnitudeStrategy
+from bead.simulation.strategies.multi_select import MultiSelectStrategy
+from bead.simulation.strategies.ordinal_scale import OrdinalScaleStrategy
+
+if TYPE_CHECKING:
+    from bead.config.simulation import SimulatedAnnotatorConfig
+    from bead.items.item import Item
+    from bead.items.item_template import ItemTemplate
+
+
+class DistanceBasedAnnotator(SimulatedAnnotator):
+    """Annotator using embedding distances for decisions.
+
+    Uses embeddings from Item.model_outputs to compute similarity/distance
+    metrics, then makes decisions based on those distances.
+
+    For forced choice, selects option with lowest distance (highest similarity).
+    For ordinal scales, maps distance to scale values.
+    For binary, thresholds distance.
+
+    Parameters
+    ----------
+    config
+        Configuration for annotator.
+
+    Examples
+    --------
+    >>> from bead.config.simulation import SimulatedAnnotatorConfig, NoiseModelConfig
+    >>> config = SimulatedAnnotatorConfig(
+    ...     strategy="distance",
+    ...     model_output_key="embedding",
+    ...     noise_model=NoiseModelConfig(noise_type="none")
+    ... )
+    >>> annotator = DistanceBasedAnnotator(config)
+    >>> # judgment = annotator.annotate(item, template)
+    """
+
+    def __init__(self, config: SimulatedAnnotatorConfig) -> None:
+        super().__init__(config)
+
+        # initialize strategies for different task types;
+        # use same strategies as LM-based, but extract embeddings instead of LM scores
+        self.strategies = {
+            "forced_choice": ForcedChoiceStrategy(),
+            "binary": BinaryStrategy(),
+            "ordinal_scale": OrdinalScaleStrategy(),
+            "categorical": CategoricalStrategy(),
+            "magnitude": MagnitudeStrategy(),
+            "multi_select": MultiSelectStrategy(),
+            "free_text": FreeTextStrategy(),
+            "cloze": ClozeStrategy(),
+        }
+
+        # initialize noise model if configured
+        if config.noise_model.noise_type == "temperature":
+            from bead.simulation.noise_models.temperature import (  # noqa: PLC0415
+                TemperatureNoiseModel,
+            )
+
+            self.noise_model = TemperatureNoiseModel(
+                temperature=config.noise_model.temperature
+            )
+        elif config.noise_model.noise_type == "none":
+            self.noise_model = None
+        else:
+            # default: no noise
+            self.noise_model = None
+
+    def annotate(
+        self, item: Item, item_template: ItemTemplate
+    ) -> str | int | float | bool | list[str]:
+        """Generate annotation using embedding distances.
+
+        Parameters
+        ----------
+        item : Item
+            Item to annotate.
+        item_template : ItemTemplate
+            Template defining task.
+
+        Returns
+        -------
+        str | int | float | bool | list[str]
+            Annotation (format depends on task type).
+
+        Notes
+        -----
+        For distance-based decisions, we convert embeddings to scores:
+        - Cosine similarity ranges from -1 (opposite) to 1 (identical)
+        - We convert to "score" by: score = similarity * 10
+        - This allows reuse of existing strategies
+        """
+        # get strategy for task type
+        strategy = self.get_strategy(item_template.task_type)
+
+        # validate item
+        strategy.validate_item(item, item_template)
+
+        # for distance-based, we need to convert embeddings to scores;
+        # this is a simplified approach that relies on strategies to extract
+        # embeddings and treat them as scores (strategies will use model_output_key)
+
+        # generate base response
+        response = strategy.simulate_response(
+            item=item,
+            item_template=item_template,
+            model_output_key=self.config.model_output_key,
+            rng=self.rng,
+        )
+
+        # apply noise model if configured
+        if self.noise_model is not None:
+            response = self.noise_model.apply(
+                value=response,
+                context={
+                    "item": item,
+                    "template": item_template,
+                    "strategy": strategy,
+                },
+                rng=self.rng,
+            )
+
+        return response