bead 0.1.0__py3-none-any.whl

bead/simulation/noise_models/systematic.py

@@ -0,0 +1,132 @@
+"""Systematic bias noise model."""
+
+from __future__ import annotations
+
+import numpy as np
+
+from bead.simulation.noise_models.base import NoiseModel
+
+
+class SystematicNoiseModel(NoiseModel):
+    """Systematic bias noise model.
+
+    Adds consistent biases to responses:
+    - length: Prefer shorter/longer options
+    - frequency: Prefer common/rare words
+    - position: Prefer first/last option
+    - endpoint: Prefer endpoints on ordinal scales
+    - midpoint: Prefer midpoint on ordinal scales
+
+    Parameters
+    ----------
+    bias_type
+        Type of bias ("length", "frequency", "position", "endpoint", "midpoint").
+        Default: "position".
+    bias_strength
+        Strength of bias (0.0-1.0). Default: 0.0.
+
+    Examples
+    --------
+    >>> noise_model = SystematicNoiseModel(bias_type="position", bias_strength=0.3)
+    >>> # Adds 30% bias toward first option in forced choice
+    """
+
+    def __init__(self, bias_type: str = "position", bias_strength: float = 0.0) -> None:
+        self.bias_type = bias_type
+        self.bias_strength = bias_strength
+
+    def apply(
+        self,
+        value: str | int | float | bool | list[str],
+        context: dict[str, str | int | float | bool | list[str]],
+        rng: np.random.RandomState,
+    ) -> str | int | float | bool | list[str]:
+        """Apply systematic bias.
+
+        Parameters
+        ----------
+        value
+            Original value.
+        context : dict
+            Context with item, template, strategy.
+        rng : np.random.RandomState
+            Random number generator.
+
+        Returns
+        -------
+        str | int | float | bool | list[str]
+            Value with bias applied.
+        """
+        if self.bias_strength == 0.0:
+            return value
+
+        strategy = context.get("strategy")
+        template = context.get("template")
+
+        if not strategy or not template:
+            return value
+
+        task_type = strategy.supported_task_type
+
+        # position bias for choice tasks
+        is_choice_task = task_type in ["forced_choice", "categorical"]
+        if is_choice_task and self.bias_type == "position":
+            return self._apply_position_bias(value, template, rng)
+
+        # endpoint/midpoint bias for ordinal scales
+        elif task_type == "ordinal_scale":
+            if self.bias_type == "endpoint":
+                return self._apply_endpoint_bias(value, template, rng)
+            elif self.bias_type == "midpoint":
+                return self._apply_midpoint_bias(value, template, rng)
+
+        # no bias for other combinations
+        return value
+
+    def _apply_position_bias(
+        self, value: str, template: str, rng: np.random.RandomState
+    ) -> str:
+        """Apply position bias to choice tasks."""
+        options = template.task_spec.options
+        if not options or len(options) < 2:
+            return value
+
+        # bias toward first option
+        if rng.random() < self.bias_strength:
+            return options[0]
+
+        return value
+
+    def _apply_endpoint_bias(
+        self, value: int, template: str, rng: np.random.RandomState
+    ) -> int:
+        """Apply endpoint bias to ordinal scales."""
+        scale_bounds = template.task_spec.scale_bounds
+        if scale_bounds is not None:
+            min_val, max_val = scale_bounds
+        else:
+            min_val, max_val = 1, 7
+
+        # bias toward endpoints (min or max)
+        if rng.random() < self.bias_strength:
+            return min_val if rng.random() < 0.5 else max_val
+
+        return value
+
+    def _apply_midpoint_bias(
+        self, value: int, template: str, rng: np.random.RandomState
+    ) -> int:
+        """Apply midpoint bias to ordinal scales."""
+        scale_bounds = template.task_spec.scale_bounds
+        if scale_bounds is not None:
+            min_val, max_val = scale_bounds
+        else:
+            min_val, max_val = 1, 7
+
+        midpoint = (min_val + max_val) // 2
+
+        # bias toward midpoint
+        if rng.random() < self.bias_strength:
+            return midpoint
+
+        return value

bead/simulation/noise_models/temperature.py

@@ -0,0 +1,86 @@
+"""Temperature-based noise model."""
+
+from __future__ import annotations
+
+import numpy as np
+
+from bead.simulation.noise_models.base import NoiseModel
+
+
+class TemperatureNoiseModel(NoiseModel):
+    """Temperature scaling for probability distributions.
+
+    Scales logits or probabilities by temperature before sampling:
+        - temperature < 1.0: More deterministic (sharper distribution)
+        - temperature = 1.0: No change
+        - temperature > 1.0: More random (flatter distribution)
+
+    For forced choice, modifies the softmax:
+        P_i = exp(score_i / T) / sum(exp(score_j / T))
+
+    Parameters
+    ----------
+    temperature
+        Temperature scaling factor (> 0). Default: 1.0.
+
+    Raises
+    ------
+    ValueError
+        If temperature <= 0.
+
+    Examples
+    --------
+    >>> noise_model = TemperatureNoiseModel(temperature=2.0)
+    >>> # More random decisions
+    """
+
+    def __init__(self, temperature: float = 1.0) -> None:
+        if temperature <= 0:
+            msg = "Temperature must be positive"
+            raise ValueError(msg)
+        self.temperature = temperature
+
+    def apply(
+        self,
+        value: str | int | float | list[str],
+        context: dict[str, str | int | float | bool | list[str]],
+        rng: np.random.RandomState,
+    ) -> str | int | float | list[str]:
+        """Apply temperature scaling.
+
+        For forced_choice, re-samples with scaled probabilities.
+        For ordinal_scale, adds scaled noise to value.
+
+        Parameters
+        ----------
+        value : str | int | float | list[str]
+            Original value (choice, rating, etc.).
+        context : dict[str, str | int | float | bool | list[str]]
+            Context with item, template, strategy.
+        rng : np.random.RandomState
+            Random number generator.
+
+        Returns
+        -------
+        str | int | float | list[str]
+            Value with temperature applied.
+        """
+        strategy = context.get("strategy")
+
+        if strategy and hasattr(strategy, "supported_task_type"):
+            task_type = strategy.supported_task_type
+
+            if task_type == "forced_choice":
+                # for forced choice, temperature is already handled in strategy
+                # by applying it to the softmax computation; return value as-is
+                return value
+
+            elif task_type == "ordinal_scale":
+                # for ordinal, add temperature-scaled gaussian noise
+                if isinstance(value, int | float):
+                    noise = rng.normal(0, self.temperature * 0.5)
+                    return value + noise
+                return value
+
+        # default: no modification
+        return value

bead/simulation/runner.py

@@ -0,0 +1,144 @@
+"""Simulation runner for orchestrating multi-annotator simulations."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from bead.config.simulation import SimulationRunnerConfig
+    from bead.items.item import Item
+    from bead.items.item_template import ItemTemplate
+
+
+class SimulationRunner:
+    """Orchestrates multi-annotator simulation.
+
+    Can simulate:
+    - Multiple independent annotators
+    - Correlated annotators (shared noise component)
+    - Mixed strategies (some LM-based, some random)
+
+    Parameters
+    ----------
+    config
+        Configuration for simulation.
+
+    Examples
+    --------
+    >>> from bead.config.simulation import (  # doctest: +SKIP
+    ...     SimulationRunnerConfig,
+    ...     SimulatedAnnotatorConfig,
+    ... )
+    >>> config = SimulationRunnerConfig(  # doctest: +SKIP
+    ...     annotator_configs=[
+    ...         SimulatedAnnotatorConfig(strategy="lm_score", random_state=1),
+    ...         SimulatedAnnotatorConfig(strategy="lm_score", random_state=2),
+    ...     ],
+    ...     n_annotators=2
+    ... )
+    >>> runner = SimulationRunner(config)
+    >>> # results = runner.run(items, templates)
+    """
+
+    def __init__(self, config: SimulationRunnerConfig) -> None:
+        self.config = config
+
+        # create annotators from configs
+        from bead.simulation.annotators.base import (  # noqa: PLC0415
+            SimulatedAnnotator,
+        )
+
+        self.annotators = [
+            SimulatedAnnotator.from_config(cfg) for cfg in config.annotator_configs
+        ]
+
+        # if n_annotators > len(annotator_configs), replicate first config
+        if config.n_annotators > len(self.annotators):
+            base_config = config.annotator_configs[0]
+            for i in range(len(self.annotators), config.n_annotators):
+                # create new config with different seed
+                new_config = base_config.model_copy()
+                new_config.random_state = (base_config.random_state or 0) + i
+                self.annotators.append(SimulatedAnnotator.from_config(new_config))
+
+    def run(
+        self,
+        items: list[Item],
+        templates: list[ItemTemplate] | ItemTemplate,
+    ) -> dict[str, list[str | int | float | list[str]]]:
+        """Run simulation with all annotators.
+
+        Parameters
+        ----------
+        items : list[Item]
+            Items to annotate.
+        templates : list[ItemTemplate] | ItemTemplate
+            Templates (one per item or shared).
+
+        Returns
+        -------
+        dict[str, list[str | int | float | list[str]]]
+            Results: {
+                "item_ids": [...],
+                "annotator_0": [...],
+                "annotator_1": [...],
+                ...
+            }
+        """
+        # collect annotations from each annotator
+        results: dict[str, list[str | int | float | list[str]]] = {
+            "item_ids": [str(item.id) for item in items]
+        }
+
+        for i, annotator in enumerate(self.annotators):
+            annotations = annotator.annotate_batch(items, templates)
+            results[f"annotator_{i}"] = [annotations[str(item.id)] for item in items]
+
+        # save if configured
+        if self.config.save_path:
+            self.save_results(results)
+
+        return results
+
+    def save_results(
+        self, results: dict[str, list[str | int | float | list[str]]]
+    ) -> None:
+        """Save results to file.
+
+        Parameters
+        ----------
+        results : dict[str, list[str | int | float | list[str]]]
+            Simulation results.
+        """
+        if self.config.save_path is None:
+            msg = "save_path not configured"
+            raise ValueError(msg)
+
+        path = Path(self.config.save_path)
+        path.parent.mkdir(parents=True, exist_ok=True)
+
+        if self.config.output_format == "jsonl":
+            # write JSONL format
+            with open(path, "w") as f:
+                for i in range(len(results["item_ids"])):
+                    row = {
+                        "item_id": results["item_ids"][i],
+                        **{
+                            key: results[key][i] for key in results if key != "item_ids"
+                        },
+                    }
+                    f.write(json.dumps(row) + "\n")
+
+        elif self.config.output_format == "dict":
+            # write JSON format
+            with open(path, "w") as f:
+                json.dump(results, f, indent=2)
+
+        elif self.config.output_format == "dataframe":
+            # write CSV format (optional dependency)
+            import pandas as pd  # noqa: PLC0415
+
+            df = pd.DataFrame(results)
+            df.to_csv(path, index=False)

bead/simulation/strategies/__init__.py

@@ -0,0 +1,23 @@
+"""Task-specific simulation strategies."""
+
+from bead.simulation.strategies.base import SimulationStrategy
+from bead.simulation.strategies.binary import BinaryStrategy
+from bead.simulation.strategies.categorical import CategoricalStrategy
+from bead.simulation.strategies.cloze import ClozeStrategy
+from bead.simulation.strategies.forced_choice import ForcedChoiceStrategy
+from bead.simulation.strategies.free_text import FreeTextStrategy
+from bead.simulation.strategies.magnitude import MagnitudeStrategy
+from bead.simulation.strategies.multi_select import MultiSelectStrategy
+from bead.simulation.strategies.ordinal_scale import OrdinalScaleStrategy
+
+__all__ = [
+    "SimulationStrategy",
+    "BinaryStrategy",
+    "CategoricalStrategy",
+    "ClozeStrategy",
+    "ForcedChoiceStrategy",
+    "FreeTextStrategy",
+    "MagnitudeStrategy",
+    "MultiSelectStrategy",
+    "OrdinalScaleStrategy",
+]

bead/simulation/strategies/base.py

@@ -0,0 +1,123 @@
+"""Base class for simulation strategies."""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import TYPE_CHECKING
+
+import numpy as np
+
+if TYPE_CHECKING:
+    from bead.items.item import Item
+    from bead.items.item_template import ItemTemplate
+
+
+class SimulationStrategy(ABC):
+    """Abstract base for task-specific simulation strategies.
+
+    Each strategy handles one task type (forced_choice, ordinal_scale, etc.)
+    and converts model outputs into appropriate responses.
+
+    Strategies should:
+    1. Validate item compatibility with task type
+    2. Extract relevant model outputs
+    3. Generate response in correct format for task
+    4. Handle missing model outputs gracefully
+    """
+
+    @property
+    @abstractmethod
+    def supported_task_type(self) -> str:
+        """Return supported task type (e.g., 'forced_choice').
+
+        Returns
+        -------
+        str
+            Task type identifier.
+        """
+
+    @abstractmethod
+    def validate_item(self, item: Item, item_template: ItemTemplate) -> None:
+        """Validate item is compatible with this strategy.
+
+        Parameters
+        ----------
+        item : Item
+            Item to validate.
+        item_template : ItemTemplate
+            Template defining task structure.
+
+        Raises
+        ------
+        ValueError
+            If item incompatible with this strategy.
+        """
+
+    @abstractmethod
+    def simulate_response(
+        self,
+        item: Item,
+        item_template: ItemTemplate,
+        model_output_key: str,
+        rng: np.random.RandomState,
+    ) -> str | int | float | list[str]:
+        """Generate simulated response for item.
+
+        Parameters
+        ----------
+        item : Item
+            Item to respond to.
+        item_template : ItemTemplate
+            Template defining task structure.
+        model_output_key : str
+            Key to extract from model outputs.
+        rng : np.random.RandomState
+            Random number generator.
+
+        Returns
+        -------
+        str | int | float | list[str]
+            Simulated response (format depends on task type).
+        """
+
+    def extract_model_outputs(
+        self, item: Item, key: str, required_count: int | None = None
+    ) -> list[float] | None:
+        """Extract model outputs from item.
+
+        Parameters
+        ----------
+        item : Item
+            Item to extract from.
+        key : str
+            Key to look for.
+        required_count : int | None
+            Expected number of outputs.
+
+        Returns
+        -------
+        list[float] | None
+            Extracted values or None if missing.
+        """
+        # try model_outputs first
+        values: list[float] = []
+        if hasattr(item, "model_outputs"):
+            for output in item.model_outputs:
+                if output.operation == key:
+                    values.append(float(output.output))
+
+        # try item_metadata as fallback
+        if not values and hasattr(item, "item_metadata"):
+            # look for keys like "lm_score1", "lm_score2", etc.
+            for i in range(1, (required_count or 10) + 1):
+                key_with_num = f"{key}{i}"
+                if key_with_num in item.item_metadata:
+                    values.append(float(item.item_metadata[key_with_num]))
+
+        if not values:
+            return None
+
+        if required_count and len(values) != required_count:
+            return None
+
+        return values

bead/simulation/strategies/binary.py

@@ -0,0 +1,103 @@
+"""Binary choice simulation strategy."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import numpy as np
+
+from bead.simulation.strategies.base import SimulationStrategy
+
+if TYPE_CHECKING:
+    from bead.items.item import Item
+    from bead.items.item_template import ItemTemplate
+
+
+class BinaryStrategy(SimulationStrategy):
+    """Strategy for binary tasks (yes/no, true/false).
+
+    Uses model outputs to compute probability of "yes" response,
+    then samples from Bernoulli distribution.
+
+    For binary tasks with LM score:
+        P(yes) = sigmoid(score / temperature)
+
+    Examples
+    --------
+    >>> strategy = BinaryStrategy()
+    >>> strategy.supported_task_type
+    'binary'
+    """
+
+    @property
+    def supported_task_type(self) -> str:
+        """Return 'binary'.
+
+        Returns
+        -------
+        str
+            Task type identifier.
+        """
+        return "binary"
+
+    def validate_item(self, item: Item, item_template: ItemTemplate) -> None:
+        """Validate item for binary choice.
+
+        Checks:
+        - task_type is 'binary'
+        - Item has appropriate model outputs OR can fall back
+
+        Parameters
+        ----------
+        item : Item
+            Item to validate.
+        item_template : ItemTemplate
+            Template defining task.
+
+        Raises
+        ------
+        ValueError
+            If validation fails.
+        """
+        if item_template.task_type != "binary":
+            msg = f"Expected task_type 'binary', got '{item_template.task_type}'"
+            raise ValueError(msg)
+
+    def simulate_response(
+        self,
+        item: Item,
+        item_template: ItemTemplate,
+        model_output_key: str,
+        rng: np.random.RandomState,
+    ) -> bool:
+        """Generate binary response.
+
+        Parameters
+        ----------
+        item : Item
+            Item to respond to.
+        item_template : ItemTemplate
+            Template defining task.
+        model_output_key : str
+            Key for model outputs (e.g., "lm_score").
+        rng : np.random.RandomState
+            Random number generator.
+
+        Returns
+        -------
+        bool
+            Binary response (True/False).
+        """
+        # extract model output (expecting single score)
+        scores = self.extract_model_outputs(item, model_output_key, required_count=1)
+
+        if scores is None:
+            # fallback to uniform random (50/50)
+            return bool(rng.rand() > 0.5)
+
+        # convert score to probability using sigmoid: 1 / (1 + exp(-x))
+        score = scores[0]
+        prob_yes = 1.0 / (1.0 + np.exp(-score))
+
+        # sample from Bernoulli
+        return bool(rng.rand() < prob_yes)