bead 0.1.0__py3-none-any.whl

bead/simulation/strategies/categorical.py

@@ -0,0 +1,123 @@
+"""Categorical choice simulation strategy."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import numpy as np
+
+from bead.simulation.strategies.base import SimulationStrategy
+
+if TYPE_CHECKING:
+    from bead.items.item import Item
+    from bead.items.item_template import ItemTemplate
+
+
+class CategoricalStrategy(SimulationStrategy):
+    """Strategy for categorical tasks (unordered multi-class).
+
+    Similar to forced_choice but for unordered categories (e.g., NLI labels,
+    sentiment classes). Uses softmax over model outputs.
+
+    For categorical with LM scores:
+        P(category_i) = softmax([score_1, ..., score_n] / temperature)[i]
+
+    Examples
+    --------
+    >>> strategy = CategoricalStrategy()
+    >>> strategy.supported_task_type
+    'categorical'
+    """
+
+    @property
+    def supported_task_type(self) -> str:
+        """Return 'categorical'.
+
+        Returns
+        -------
+        str
+            Task type identifier.
+        """
+        return "categorical"
+
+    def validate_item(self, item: Item, item_template: ItemTemplate) -> None:
+        """Validate item for categorical choice.
+
+        Checks:
+        - task_type is 'categorical'
+        - task_spec.options is defined
+        - At least 2 options available
+
+        Parameters
+        ----------
+        item : Item
+            Item to validate.
+        item_template : ItemTemplate
+            Template defining task.
+
+        Raises
+        ------
+        ValueError
+            If validation fails.
+        """
+        if item_template.task_type != "categorical":
+            msg = f"Expected task_type 'categorical', got '{item_template.task_type}'"
+            raise ValueError(msg)
+
+        if not item_template.task_spec.options:
+            msg = "task_spec.options must be defined for categorical"
+            raise ValueError(msg)
+
+        if len(item_template.task_spec.options) < 2:
+            msg = "categorical requires at least 2 options"
+            raise ValueError(msg)
+
+    def simulate_response(
+        self,
+        item: Item,
+        item_template: ItemTemplate,
+        model_output_key: str,
+        rng: np.random.RandomState,
+    ) -> str:
+        """Generate categorical response.
+
+        Parameters
+        ----------
+        item : Item
+            Item to respond to.
+        item_template : ItemTemplate
+            Template defining task.
+        model_output_key : str
+            Key for model outputs (e.g., "lm_score").
+        rng : np.random.RandomState
+            Random number generator.
+
+        Returns
+        -------
+        str
+            Chosen category name.
+        """
+        options = item_template.task_spec.options
+        if options is None:
+            msg = "task_spec.options must be defined"
+            raise ValueError(msg)
+
+        n_options = len(options)
+
+        # extract model outputs for each category
+        scores = self.extract_model_outputs(item, model_output_key, n_options)
+
+        if scores is None:
+            # fallback to uniform random
+            choice_idx = rng.randint(0, n_options)
+            return options[choice_idx]
+
+        # convert scores to probabilities using softmax
+        scores_array = np.array(scores)
+        exp_scores = np.exp(scores_array - np.max(scores_array))  # numerical stability
+        probs = exp_scores / np.sum(exp_scores)
+
+        # sample from distribution
+        choice_idx = rng.choice(n_options, p=probs)
+
+        return options[choice_idx]

bead/simulation/strategies/cloze.py

@@ -0,0 +1,224 @@
+"""Cloze (fill-in-the-blank) simulation strategy using MLM scores."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import numpy as np
+
+from bead.simulation.strategies.base import SimulationStrategy
+
+if TYPE_CHECKING:
+    from bead.items.item import Item
+    from bead.items.item_template import ItemTemplate
+
+__all__ = ["ClozeStrategy"]
+
+
+class ClozeStrategy(SimulationStrategy):
+    """MLM-based strategy for cloze (fill-in-the-blank) tasks.
+
+    Uses masked language model scores to select fillers for unfilled slots.
+    For constrained slots (with specific options), selects highest-scoring option.
+    For unconstrained slots, uses rendered_elements or metadata as fallback.
+
+    The strategy expects model outputs to contain MLM scores for each slot,
+    stored as separate ModelOutput instances with operation="mlm_score" and
+    inputs containing {"slot_name": slot_name, "candidate": candidate_value}.
+
+    Examples
+    --------
+    >>> from bead.simulation.strategies.cloze import ClozeStrategy
+    >>> strategy = ClozeStrategy()
+    >>> # item with unfilled_slots and model_outputs with MLM scores
+    >>> # response = strategy.simulate_response(item, template, "mlm_score", rng)
+    >>> # Returns: {"determiner": "the", "verb": "chased", "object": "mouse"}
+    """
+
+    @property
+    def supported_task_type(self) -> str:
+        """Get supported task type.
+
+        Returns
+        -------
+        str
+            Always returns "cloze".
+        """
+        return "cloze"
+
+    def validate_item(self, item: Item, item_template: ItemTemplate) -> None:
+        """Validate item is compatible with cloze strategy.
+
+        Parameters
+        ----------
+        item : Item
+            Item to validate.
+        item_template : ItemTemplate
+            Template defining task.
+
+        Raises
+        ------
+        ValueError
+            If validation fails.
+        """
+        if item_template.task_type != "cloze":
+            msg = f"Expected task_type 'cloze', got '{item_template.task_type}'"
+            raise ValueError(msg)
+
+        if not item.unfilled_slots:
+            raise ValueError("cloze task requires at least one unfilled slot")
+
+    def simulate_response(
+        self,
+        item: Item,
+        item_template: ItemTemplate,
+        model_output_key: str,
+        rng: np.random.RandomState,
+    ) -> dict[str, str]:
+        """Simulate cloze response using MLM scores.
+
+        For each unfilled slot, selects the filler with highest MLM score.
+        Falls back to random selection or metadata if MLM scores unavailable.
+
+        Parameters
+        ----------
+        item : Item
+            Item to annotate.
+        item_template : ItemTemplate
+            Template defining task constraints.
+        model_output_key : str
+            Key identifying which model outputs to use (e.g., "mlm_score").
+        rng : np.random.RandomState
+            Random number generator for stochasticity.
+
+        Returns
+        -------
+        dict[str, str]
+            Dictionary mapping slot names to selected fillers.
+
+        Examples
+        --------
+        >>> response = {"determiner": "the", "verb": "chased", "object": "mouse"}
+        """
+        response = {}
+
+        for slot in item.unfilled_slots:
+            slot_name = slot.slot_name
+
+            # try to get MLM scores for this slot
+            slot_scores = self._get_slot_scores(item, slot_name, model_output_key)
+
+            if slot_scores:
+                # select filler with highest score (with softmax sampling)
+                fillers = list(slot_scores.keys())
+                scores = np.array(list(slot_scores.values()))
+
+                # apply softmax to convert scores to probabilities
+                exp_scores = np.exp(scores - np.max(scores))  # numerical stability
+                probs = exp_scores / np.sum(exp_scores)
+
+                # sample from distribution
+                selected_idx = rng.choice(len(fillers), p=probs)
+                response[slot_name] = fillers[selected_idx]
+            else:
+                # fallback: use ground truth if available, else placeholder
+                response[slot_name] = self._get_fallback_filler(item, slot_name, rng)
+
+        return response
+
+    def _get_slot_scores(
+        self, item: Item, slot_name: str, model_output_key: str
+    ) -> dict[str, float]:
+        """Extract MLM scores for a specific slot.
+
+        Looks for ModelOutput instances where:
+        - operation matches model_output_key (e.g., "mlm_score")
+        - inputs contains {"slot_name": slot_name}
+        - inputs contains "candidate" (the filler being scored)
+        - output is the MLM score
+
+        Parameters
+        ----------
+        item : Item
+            Item containing model outputs.
+        slot_name : str
+            Name of the slot to get scores for.
+        model_output_key : str
+            Operation type to filter by.
+
+        Returns
+        -------
+        dict[str, float]
+            Mapping from candidate fillers to MLM scores.
+        """
+        scores = {}
+
+        for model_output in item.model_outputs:
+            if model_output.operation != model_output_key:
+                continue
+
+            inputs = model_output.inputs
+
+            # check if this output is for our slot
+            if inputs.get("slot_name") != slot_name:
+                continue
+
+            candidate = inputs.get("candidate")
+            if candidate is None:
+                continue
+
+            # extract score
+            score = model_output.output
+            if isinstance(score, int | float):
+                scores[str(candidate)] = float(score)
+
+        return scores
+
+    def _get_fallback_filler(
+        self, item: Item, slot_name: str, rng: np.random.RandomState
+    ) -> str:
+        """Get fallback filler when MLM scores unavailable.
+
+        Priority:
+        1. Ground truth from item_metadata["ground_truth"][slot_name]
+        2. Random common filler based on slot name pattern
+        3. Generic placeholder
+
+        Parameters
+        ----------
+        item : Item
+            Item to get fallback from.
+        slot_name : str
+            Slot name.
+        rng : np.random.RandomState
+            Random number generator.
+
+        Returns
+        -------
+        str
+            Fallback filler.
+        """
+        # try ground truth
+        if hasattr(item, "item_metadata") and item.item_metadata:
+            ground_truth = item.item_metadata.get("ground_truth")
+            if isinstance(ground_truth, dict) and slot_name in ground_truth:
+                return str(ground_truth[slot_name])
+
+        # common fallbacks by slot name patterns
+        fallback_options = {
+            "determiner": ["the", "a", "an", "this", "that"],
+            "verb": ["is", "was", "has", "can", "will"],
+            "noun": ["thing", "person", "place", "time", "way"],
+            "adjective": ["good", "new", "old", "big", "small"],
+            "adverb": ["very", "well", "just", "now", "here"],
+            "preposition": ["in", "on", "at", "to", "for"],
+        }
+
+        # match slot name to category
+        slot_lower = slot_name.lower()
+        for category, options in fallback_options.items():
+            if category in slot_lower:
+                return str(rng.choice(options))
+
+        # generic fallback
+        return f"[{slot_name}]"

bead/simulation/strategies/forced_choice.py

@@ -0,0 +1,127 @@
+"""Forced choice simulation strategy."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import numpy as np
+
+from bead.simulation.strategies.base import SimulationStrategy
+
+if TYPE_CHECKING:
+    from bead.items.item import Item
+    from bead.items.item_template import ItemTemplate
+
+
+class ForcedChoiceStrategy(SimulationStrategy):
+    """Strategy for forced_choice tasks (n-AFC).
+
+    Handles 2AFC, 3AFC, 4AFC, etc. Uses model outputs to compute
+    preference probabilities, then samples categorically.
+
+    For 2AFC with LM scores:
+        P(choose A) = sigmoid((score_A - score_B) / temperature)
+
+    For n-AFC with LM scores:
+        P(choose i) = softmax([score_1, ..., score_n] / temperature)[i]
+
+    Examples
+    --------
+    >>> strategy = ForcedChoiceStrategy()
+    >>> strategy.supported_task_type
+    'forced_choice'
+    """
+
+    @property
+    def supported_task_type(self) -> str:
+        """Return 'forced_choice'.
+
+        Returns
+        -------
+        str
+            Task type identifier.
+        """
+        return "forced_choice"
+
+    def validate_item(self, item: Item, item_template: ItemTemplate) -> None:
+        """Validate item for forced choice.
+
+        Checks:
+        - task_type is 'forced_choice'
+        - task_spec.options is defined
+        - Item has appropriate model outputs OR can fall back
+
+        Parameters
+        ----------
+        item : Item
+            Item to validate.
+        item_template : ItemTemplate
+            Template defining task.
+
+        Raises
+        ------
+        ValueError
+            If validation fails.
+        """
+        if item_template.task_type != "forced_choice":
+            msg = f"Expected task_type 'forced_choice', got '{item_template.task_type}'"
+            raise ValueError(msg)
+
+        if not item_template.task_spec.options:
+            msg = "task_spec.options must be defined for forced_choice"
+            raise ValueError(msg)
+
+        if len(item_template.task_spec.options) < 2:
+            msg = "forced_choice requires at least 2 options"
+            raise ValueError(msg)
+
+    def simulate_response(
+        self,
+        item: Item,
+        item_template: ItemTemplate,
+        model_output_key: str,
+        rng: np.random.RandomState,
+    ) -> str:
+        """Generate forced choice response.
+
+        Parameters
+        ----------
+        item : Item
+            Item to respond to.
+        item_template : ItemTemplate
+            Template defining task.
+        model_output_key : str
+            Key for model outputs (e.g., "lm_score").
+        rng : np.random.RandomState
+            Random number generator.
+
+        Returns
+        -------
+        str
+            Chosen option name.
+        """
+        options = item_template.task_spec.options
+        if options is None:
+            msg = "task_spec.options must be defined"
+            raise ValueError(msg)
+
+        n_options = len(options)
+
+        # extract model outputs for each option
+        scores = self.extract_model_outputs(item, model_output_key, n_options)
+
+        if scores is None:
+            # fallback to uniform random
+            choice_idx = rng.randint(0, n_options)
+            return options[choice_idx]
+
+        # convert scores to probabilities using softmax
+        # (will be scaled by noise model later)
+        scores_array = np.array(scores)
+        exp_scores = np.exp(scores_array - np.max(scores_array))  # numerical stability
+        probs = exp_scores / np.sum(exp_scores)
+
+        # sample from distribution
+        choice_idx = rng.choice(n_options, p=probs)
+
+        return options[choice_idx]

bead/simulation/strategies/free_text.py

@@ -0,0 +1,105 @@
+"""Free text simulation strategy."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import numpy as np
+
+from bead.simulation.strategies.base import SimulationStrategy
+
+if TYPE_CHECKING:
+    from bead.items.item import Item
+    from bead.items.item_template import ItemTemplate
+
+
+class FreeTextStrategy(SimulationStrategy):
+    """Strategy for free_text tasks.
+
+    Handles free text generation using rule-based approaches.
+    For simulations, this typically:
+    - Extracts text from rendered_elements
+    - Uses templates if provided
+    - Falls back to simple defaults
+
+    Note: This is a simplified implementation for simulation purposes.
+    For realistic free text generation, consider using LLMs.
+
+    Examples
+    --------
+    >>> strategy = FreeTextStrategy()
+    >>> strategy.supported_task_type
+    'free_text'
+    """
+
+    @property
+    def supported_task_type(self) -> str:
+        """Return 'free_text'."""
+        return "free_text"
+
+    def validate_item(self, item: Item, item_template: ItemTemplate) -> None:
+        """Validate item for free text.
+
+        Checks:
+        - task_type is 'free_text'
+
+        Parameters
+        ----------
+        item : Item
+            Item to validate.
+        item_template : ItemTemplate
+            Template defining task.
+
+        Raises
+        ------
+        ValueError
+            If validation fails.
+        """
+        if item_template.task_type != "free_text":
+            raise ValueError(
+                f"Expected task_type 'free_text', got '{item_template.task_type}'"
+            )
+
+    def simulate_response(
+        self,
+        item: Item,
+        item_template: ItemTemplate,
+        model_output_key: str,
+        rng: np.random.RandomState,
+    ) -> str:
+        """Generate free text response.
+
+        Parameters
+        ----------
+        item : Item
+            Item to respond to.
+        item_template : ItemTemplate
+            Template defining task.
+        model_output_key : str
+            Key for model outputs (unused for free text).
+        rng : np.random.RandomState
+            Random number generator.
+
+        Returns
+        -------
+        str
+            Generated text response.
+        """
+        # check if there's a ground truth response we can use
+        if hasattr(item, "item_metadata") and "response" in item.item_metadata:
+            return str(item.item_metadata["response"])
+
+        # check for text template
+        if hasattr(item, "item_metadata") and "response_template" in item.item_metadata:
+            return str(item.item_metadata["response_template"])
+
+        # try to extract from rendered elements
+        if hasattr(item, "rendered_elements") and item.rendered_elements:
+            # get first text element as fallback
+            for value in item.rendered_elements.values():
+                if len(value) > 0:
+                    # return first non-empty string
+                    return value
+
+        # final fallback: generic response
+        return "No response"

bead/simulation/strategies/magnitude.py

@@ -0,0 +1,116 @@
+"""Magnitude estimation simulation strategy."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import numpy as np
+
+from bead.simulation.strategies.base import SimulationStrategy
+
+if TYPE_CHECKING:
+    from bead.items.item import Item
+    from bead.items.item_template import ItemTemplate
+
+
+class MagnitudeStrategy(SimulationStrategy):
+    """Strategy for magnitude estimation tasks.
+
+    Handles unbounded numeric magnitude estimation. Converts model outputs
+    (typically LM scores) to positive magnitude values.
+
+    For LM scores (typically negative log probabilities):
+        magnitude = exp(-score / scale_factor)
+
+    This maps:
+        - Better scores (less negative) -> larger magnitudes
+        - Worse scores (more negative) -> smaller magnitudes
+
+    Parameters
+    ----------
+    scale_factor
+        Scaling factor for converting scores to magnitudes.
+        Higher values produce more variation. Default: 10.0.
+
+    Examples
+    --------
+    >>> strategy = MagnitudeStrategy()
+    >>> strategy.supported_task_type
+    'magnitude'
+    """
+
+    def __init__(self, scale_factor: float = 10.0) -> None:
+        self.scale_factor = scale_factor
+
+    @property
+    def supported_task_type(self) -> str:
+        """Return 'magnitude'."""
+        return "magnitude"
+
+    def validate_item(self, item: Item, item_template: ItemTemplate) -> None:
+        """Validate item for magnitude estimation.
+
+        Checks:
+        - task_type is 'magnitude'
+        - Item has model outputs OR can fall back
+
+        Parameters
+        ----------
+        item : Item
+            Item to validate.
+        item_template : ItemTemplate
+            Template defining task.
+
+        Raises
+        ------
+        ValueError
+            If validation fails.
+        """
+        if item_template.task_type != "magnitude":
+            raise ValueError(
+                f"Expected task_type 'magnitude', got '{item_template.task_type}'"
+            )
+
+    def simulate_response(
+        self,
+        item: Item,
+        item_template: ItemTemplate,
+        model_output_key: str,
+        rng: np.random.RandomState,
+    ) -> float:
+        """Generate magnitude estimation response.
+
+        Parameters
+        ----------
+        item : Item
+            Item to respond to.
+        item_template : ItemTemplate
+            Template defining task.
+        model_output_key : str
+            Key for model outputs (e.g., "lm_score").
+        rng : np.random.RandomState
+            Random number generator.
+
+        Returns
+        -------
+        float
+            Estimated magnitude (positive value).
+        """
+        # extract model output (expect single value)
+        scores = self.extract_model_outputs(item, model_output_key, required_count=1)
+
+        if scores is None:
+            # fallback to random positive value (log-normal)
+            return float(rng.lognormal(mean=0, sigma=1))
+
+        score = scores[0]
+
+        # convert score to magnitude:
+        # for LM scores (negative), exp(-score/scale) gives positive magnitude
+        # for positive scores, use exp(score/scale)
+        if score < 0:
+            magnitude = np.exp(-score / self.scale_factor)
+        else:
+            magnitude = np.exp(score / self.scale_factor)
+
+        return float(magnitude)