bead 0.1.0__py3-none-any.whl

bead/simulation/annotators/lm_based.py

@@ -0,0 +1,114 @@
+"""LM score-based annotator."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from bead.simulation.annotators.base import SimulatedAnnotator
+from bead.simulation.noise_models.temperature import TemperatureNoiseModel
+from bead.simulation.strategies.binary import BinaryStrategy
+from bead.simulation.strategies.categorical import CategoricalStrategy
+from bead.simulation.strategies.cloze import ClozeStrategy
+from bead.simulation.strategies.forced_choice import ForcedChoiceStrategy
+from bead.simulation.strategies.free_text import FreeTextStrategy
+from bead.simulation.strategies.magnitude import MagnitudeStrategy
+from bead.simulation.strategies.multi_select import MultiSelectStrategy
+from bead.simulation.strategies.ordinal_scale import OrdinalScaleStrategy
+
+if TYPE_CHECKING:
+    from bead.config.simulation import SimulatedAnnotatorConfig
+    from bead.items.item import Item
+    from bead.items.item_template import ItemTemplate
+
+
+class LMBasedAnnotator(SimulatedAnnotator):
+    """Annotator using language model scores for decisions.
+
+    Uses LM log probabilities or scores from Item.model_outputs
+    to make informed decisions. Applies noise model for variability.
+
+    Supports all task types via pluggable strategies.
+
+    Parameters
+    ----------
+    config
+        Configuration for annotator.
+
+    Examples
+    --------
+    >>> from bead.config.simulation import SimulatedAnnotatorConfig, NoiseModelConfig
+    >>> config = SimulatedAnnotatorConfig(
+    ...     strategy="lm_score",
+    ...     model_output_key="lm_score",
+    ...     noise_model=NoiseModelConfig(noise_type="temperature", temperature=1.5)
+    ... )
+    >>> annotator = LMBasedAnnotator(config)
+    >>> # judgment = annotator.annotate(item, template)
+    """
+
+    def __init__(self, config: SimulatedAnnotatorConfig) -> None:
+        super().__init__(config)
+
+        # initialize strategies for different task types
+        self.strategies = {
+            "forced_choice": ForcedChoiceStrategy(),
+            "binary": BinaryStrategy(),
+            "ordinal_scale": OrdinalScaleStrategy(),
+            "categorical": CategoricalStrategy(),
+            "magnitude": MagnitudeStrategy(),
+            "multi_select": MultiSelectStrategy(),
+            "free_text": FreeTextStrategy(),
+            "cloze": ClozeStrategy(),
+        }
+
+        # initialize noise model
+        if config.noise_model.noise_type == "temperature":
+            self.noise_model = TemperatureNoiseModel(
+                temperature=config.noise_model.temperature
+            )
+        elif config.noise_model.noise_type == "none":
+            self.noise_model = None
+        else:
+            # default: no noise
+            self.noise_model = None
+
+    def annotate(
+        self, item: Item, item_template: ItemTemplate
+    ) -> str | int | float | list[str]:
+        """Generate annotation using LM scores.
+
+        Parameters
+        ----------
+        item : Item
+            Item to annotate.
+        item_template : ItemTemplate
+            Template defining task.
+
+        Returns
+        -------
+        str | int | float | list[str]
+            Annotation (format depends on task type).
+        """
+        # get strategy for task type
+        strategy = self.get_strategy(item_template.task_type)
+
+        # validate item
+        strategy.validate_item(item, item_template)
+
+        # generate base response
+        response = strategy.simulate_response(
+            item=item,
+            item_template=item_template,
+            model_output_key=self.config.model_output_key,
+            rng=self.rng,
+        )
+
+        # apply noise model if configured
+        if self.noise_model is not None:
+            response = self.noise_model.apply(
+                value=response,
+                context={"item": item, "template": item_template, "strategy": strategy},
+                rng=self.rng,
+            )
+
+        return response

bead/simulation/annotators/oracle.py

@@ -0,0 +1,182 @@
+"""Oracle (perfect performance) annotator."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from bead.simulation.annotators.base import SimulatedAnnotator
+from bead.simulation.annotators.random import RandomAnnotator
+
+if TYPE_CHECKING:
+    from bead.config.simulation import SimulatedAnnotatorConfig
+    from bead.items.item import Item
+    from bead.items.item_template import ItemTemplate
+
+
+class OracleAnnotator(SimulatedAnnotator):
+    """Perfect performance annotator using ground truth.
+
+    Returns ground truth labels from item.item_metadata['ground_truth'].
+    Falls back to random when ground truth is not available.
+
+    Useful for establishing upper bound on performance.
+
+    Parameters
+    ----------
+    config
+        Configuration for annotator.
+
+    Examples
+    --------
+    >>> from bead.config.simulation import SimulatedAnnotatorConfig
+    >>> config = SimulatedAnnotatorConfig(strategy="oracle", random_state=42)
+    >>> annotator = OracleAnnotator(config)
+    >>> # judgment = annotator.annotate(item, template)
+    """
+
+    def __init__(self, config: SimulatedAnnotatorConfig) -> None:
+        super().__init__(config)
+
+        # create random annotator for fallback
+        self.random_annotator = RandomAnnotator(config)
+
+    def annotate(
+        self, item: Item, item_template: ItemTemplate
+    ) -> str | int | float | bool | list[str]:
+        """Generate oracle annotation using ground truth.
+
+        Parameters
+        ----------
+        item : Item
+            Item to annotate.
+        item_template : ItemTemplate
+            Template defining task.
+
+        Returns
+        -------
+        str | int | float | bool | list[str]
+            Ground truth annotation or random fallback.
+        """
+        # try to get ground truth from item metadata
+        if hasattr(item, "item_metadata") and item.item_metadata:
+            ground_truth = item.item_metadata.get("ground_truth")
+
+            if ground_truth is not None:
+                # validate and return ground truth
+                return self._validate_ground_truth(ground_truth, item_template)
+
+        # fallback to random if no ground truth
+        return self.random_annotator.annotate(item, item_template)
+
+    def _validate_ground_truth(
+        self, ground_truth: str | int | float | bool | list[str], template: ItemTemplate
+    ) -> str | int | float | bool | list[str]:
+        """Validate ground truth against task spec.
+
+        Parameters
+        ----------
+        ground_truth
+            Ground truth value.
+        template : ItemTemplate
+            Template defining task constraints.
+
+        Returns
+        -------
+        str | int | float | bool | list[str]
+            Validated ground truth.
+
+        Raises
+        ------
+        ValueError
+            If ground truth is invalid for task type.
+        """
+        task_type = template.task_type
+
+        if task_type == "forced_choice":
+            if not isinstance(ground_truth, str):
+                msg = (
+                    f"forced_choice ground truth must be str, got {type(ground_truth)}"
+                )
+                raise ValueError(msg)
+            options = template.task_spec.options or []
+            if ground_truth not in options:
+                msg = f"Ground truth '{ground_truth}' not in options {options}"
+                raise ValueError(msg)
+            return ground_truth
+
+        elif task_type == "binary":
+            if not isinstance(ground_truth, bool):
+                msg = f"binary ground truth must be bool, got {type(ground_truth)}"
+                raise ValueError(msg)
+            return ground_truth
+
+        elif task_type == "ordinal_scale":
+            if not isinstance(ground_truth, int):
+                msg = (
+                    f"ordinal_scale ground truth must be int, got {type(ground_truth)}"
+                )
+                raise ValueError(msg)
+            scale_bounds = template.task_spec.scale_bounds
+            if scale_bounds is not None:
+                min_val, max_val = scale_bounds
+            else:
+                min_val, max_val = 1, 7
+            if not (min_val <= ground_truth <= max_val):
+                msg = f"Ground truth {ground_truth} not in range [{min_val}, {max_val}]"
+                raise ValueError(msg)
+            return ground_truth
+
+        elif task_type == "categorical":
+            if not isinstance(ground_truth, str):
+                msg = f"categorical ground truth must be str, got {type(ground_truth)}"
+                raise ValueError(msg)
+            options = template.task_spec.options or []
+            if ground_truth not in options:
+                msg = f"Ground truth '{ground_truth}' not in options {options}"
+                raise ValueError(msg)
+            return ground_truth
+
+        elif task_type == "magnitude":
+            if not isinstance(ground_truth, int | float):
+                msg = (
+                    f"magnitude ground truth must be numeric, got {type(ground_truth)}"
+                )
+                raise ValueError(msg)
+            return float(ground_truth)
+
+        elif task_type == "multi_select":
+            if not isinstance(ground_truth, list):
+                msg = (
+                    f"multi_select ground truth must be list, got {type(ground_truth)}"
+                )
+                raise ValueError(msg)
+            options = template.task_spec.options or []
+            for item_val in ground_truth:
+                if item_val not in options:
+                    msg = f"Ground truth item '{item_val}' not in options {options}"
+                    raise ValueError(msg)
+            return ground_truth
+
+        elif task_type == "free_text":
+            if not isinstance(ground_truth, str):
+                msg = f"free_text ground truth must be str, got {type(ground_truth)}"
+                raise ValueError(msg)
+            return ground_truth
+
+        elif task_type == "cloze":
+            if not isinstance(ground_truth, dict):
+                msg = f"cloze ground truth must be dict, got {type(ground_truth)}"
+                raise ValueError(msg)
+            # validate all required slots are present
+            for slot in template.unfilled_slots:
+                if slot.slot_name not in ground_truth:
+                    msg = (
+                        f"Ground truth missing slot '{slot.slot_name}' "
+                        f"(expected slots: {[s.slot_name for s in template.unfilled_slots]})"  # noqa: E501
+                    )
+                    raise ValueError(msg)
+            # return dict of slot_name -> value
+            return {k: str(v) for k, v in ground_truth.items()}
+
+        else:
+            raise ValueError(f"Unsupported task type: {task_type}")

bead/simulation/annotators/random.py

@@ -0,0 +1,181 @@
+"""Random baseline annotator."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from bead.simulation.annotators.base import SimulatedAnnotator
+
+if TYPE_CHECKING:
+    from bead.config.simulation import SimulatedAnnotatorConfig
+    from bead.items.item import Item
+    from bead.items.item_template import ItemTemplate
+
+
+class RandomAnnotator(SimulatedAnnotator):
+    """Pure random baseline annotator.
+
+    Generates random responses that respect task spec constraints
+    (options, scale ranges, etc.) but are otherwise uninformed.
+
+    Useful for establishing baseline performance.
+
+    Parameters
+    ----------
+    config
+        Configuration for annotator.
+
+    Examples
+    --------
+    >>> from bead.config.simulation import SimulatedAnnotatorConfig
+    >>> config = SimulatedAnnotatorConfig(strategy="random", random_state=42)
+    >>> annotator = RandomAnnotator(config)
+    >>> # judgment = annotator.annotate(item, template)
+    """
+
+    def __init__(self, config: SimulatedAnnotatorConfig) -> None:
+        super().__init__(config)
+
+        # no strategies or noise models needed for random
+
+    def annotate(
+        self, item: Item, item_template: ItemTemplate
+    ) -> str | int | float | bool | list[str]:
+        """Generate random annotation.
+
+        Parameters
+        ----------
+        item : Item
+            Item to annotate (ignored).
+        item_template : ItemTemplate
+            Template defining task constraints.
+
+        Returns
+        -------
+        str | int | float | bool | list[str]
+            Random annotation (format depends on task type).
+
+        Raises
+        ------
+        ValueError
+            If task type is not supported.
+        """
+        task_type = item_template.task_type
+
+        if task_type == "forced_choice":
+            return self._random_forced_choice(item_template)
+        elif task_type == "binary":
+            return self._random_binary()
+        elif task_type == "ordinal_scale":
+            return self._random_ordinal(item_template)
+        elif task_type == "categorical":
+            return self._random_categorical(item_template)
+        elif task_type == "magnitude":
+            return self._random_magnitude()
+        elif task_type == "multi_select":
+            return self._random_multi_select(item_template)
+        elif task_type == "free_text":
+            return self._random_free_text()
+        elif task_type == "cloze":
+            return self._random_cloze(item)
+        else:
+            raise ValueError(f"Unsupported task type: {task_type}")
+
+    def _random_forced_choice(self, template: ItemTemplate) -> str:
+        """Generate random forced choice response."""
+        options = template.task_spec.options or []
+        if not options:
+            raise ValueError("forced_choice requires options")
+        return str(self.rng.choice(options))
+
+    def _random_binary(self) -> bool:
+        """Generate random binary response."""
+        return bool(self.rng.choice([True, False]))
+
+    def _random_ordinal(self, template: ItemTemplate) -> int:
+        """Generate random ordinal scale response."""
+        # get scale bounds from task_spec
+        scale_bounds = template.task_spec.scale_bounds
+        if scale_bounds is not None:
+            min_val, max_val = scale_bounds
+        else:
+            min_val, max_val = 1, 7
+        return int(self.rng.randint(min_val, max_val + 1))
+
+    def _random_categorical(self, template: ItemTemplate) -> str:
+        """Generate random categorical response."""
+        options = template.task_spec.options or []
+        if not options:
+            raise ValueError("categorical requires options")
+        return str(self.rng.choice(options))
+
+    def _random_magnitude(self) -> float:
+        """Generate random magnitude response."""
+        # log-normal distribution for positive magnitudes
+        return float(self.rng.lognormal(mean=0, sigma=1))
+
+    def _random_multi_select(self, template: ItemTemplate) -> list[str]:
+        """Generate random multi-select response."""
+        options = template.task_spec.options or []
+        if not options:
+            raise ValueError("multi_select requires options")
+
+        # randomly select subset of options
+        selected = []
+        for option in options:
+            if self.rng.random() < 0.5:
+                selected.append(option)
+        return selected
+
+    def _random_free_text(self) -> str:
+        """Generate random free text response."""
+        # simple random responses
+        responses = [
+            "No response",
+            "Unclear",
+            "Cannot determine",
+            "Not applicable",
+            "Unknown",
+        ]
+        return str(self.rng.choice(responses))
+
+    def _random_cloze(self, item: Item) -> dict[str, str]:
+        """Generate random cloze response."""
+        response = {}
+
+        # common word bank for random selection
+        word_bank = [
+            "the",
+            "a",
+            "is",
+            "was",
+            "has",
+            "can",
+            "will",
+            "thing",
+            "person",
+            "place",
+            "time",
+            "way",
+            "good",
+            "new",
+            "old",
+            "big",
+            "small",
+            "very",
+            "well",
+            "just",
+            "now",
+            "here",
+            "in",
+            "on",
+            "at",
+            "to",
+            "for",
+        ]
+
+        for slot in item.unfilled_slots:
+            # randomly select a word from the bank
+            response[slot.slot_name] = str(self.rng.choice(word_bank))
+
+        return response

bead/simulation/dsl_extension/__init__.py

@@ -0,0 +1,3 @@
+"""DSL extensions for simulation-specific operations."""
+
+__all__ = []

bead/simulation/noise_models/__init__.py

@@ -0,0 +1,13 @@
+"""Noise models for simulating human variability."""
+
+from bead.simulation.noise_models.base import NoiseModel
+from bead.simulation.noise_models.random_noise import RandomNoiseModel
+from bead.simulation.noise_models.systematic import SystematicNoiseModel
+from bead.simulation.noise_models.temperature import TemperatureNoiseModel
+
+__all__ = [
+    "NoiseModel",
+    "RandomNoiseModel",
+    "SystematicNoiseModel",
+    "TemperatureNoiseModel",
+]

bead/simulation/noise_models/base.py

@@ -0,0 +1,42 @@
+"""Base class for noise models."""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+
+import numpy as np
+
+
+class NoiseModel(ABC):
+    """Abstract base for noise models.
+
+    Noise models add human-like variability to simulated responses.
+    They can:
+    - Scale probabilities by temperature
+    - Add systematic biases (length, frequency, position)
+    - Inject random noise
+    """
+
+    @abstractmethod
+    def apply(
+        self,
+        value: str | int | float | list[str],
+        context: dict[str, str | int | float | bool | list[str]],
+        rng: np.random.RandomState,
+    ) -> str | int | float | list[str]:
+        """Apply noise to value.
+
+        Parameters
+        ----------
+        value : str | int | float | list[str]
+            Original value (probability, score, choice, etc.).
+        context : dict[str, str | int | float | bool | list[str]]
+            Additional context (item, template, strategy, etc.).
+        rng : np.random.RandomState
+            Random number generator.
+
+        Returns
+        -------
+        str | int | float | list[str]
+            Value with noise applied.
+        """

bead/simulation/noise_models/random_noise.py

@@ -0,0 +1,82 @@
+"""Random noise injection model."""
+
+from __future__ import annotations
+
+import numpy as np
+
+from bead.simulation.noise_models.base import NoiseModel
+
+
+class RandomNoiseModel(NoiseModel):
+    """Random noise injection model.
+
+    Adds random noise to responses:
+    - Gaussian noise for numeric values
+    - Uniform noise for numeric values
+    - Random flipping for choice tasks
+
+    Parameters
+    ----------
+    noise_type
+        Type of noise ("gaussian" or "uniform"). Default: "gaussian".
+    strength
+        Noise strength (stddev for gaussian, range for uniform). Default: 1.0.
+
+    Examples
+    --------
+    >>> noise_model = RandomNoiseModel(noise_type="gaussian", strength=0.5)
+    >>> # Adds gaussian noise with stddev=0.5 to numeric responses
+    """
+
+    def __init__(self, noise_type: str = "gaussian", strength: float = 1.0) -> None:
+        self.noise_type = noise_type
+        self.strength = strength
+
+    def apply(
+        self,
+        value: str | int | float | bool | list[str],
+        context: dict[str, str | int | float | bool | list[str]],
+        rng: np.random.RandomState,
+    ) -> str | int | float | bool | list[str]:
+        """Apply random noise.
+
+        Parameters
+        ----------
+        value
+            Original value.
+        context : dict
+            Context with item, template, strategy.
+        rng : np.random.RandomState
+            Random number generator.
+
+        Returns
+        -------
+        str | int | float | bool | list[str]
+            Value with noise applied.
+        """
+        if self.strength == 0.0:
+            return value
+
+        # apply noise based on value type
+        if isinstance(value, int | float) and not isinstance(value, bool):
+            return self._add_numeric_noise(value, rng)
+        else:
+            # for non-numeric, return as-is
+            return value
+
+    def _add_numeric_noise(
+        self, value: int | float, rng: np.random.RandomState
+    ) -> int | float:
+        """Add noise to numeric value."""
+        if self.noise_type == "gaussian":
+            noisy_value = value + rng.normal(0, self.strength)
+        elif self.noise_type == "uniform":
+            noisy_value = value + rng.uniform(-self.strength, self.strength)
+        else:
+            noisy_value = value
+
+        # preserve type
+        if isinstance(value, int):
+            return int(round(noisy_value))
+        else:
+            return float(noisy_value)