bead 0.1.0__py3-none-any.whl

bead/config/simulation.py

@@ -0,0 +1,206 @@
+"""Simulation configuration models for the bead package."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Literal
+
+from pydantic import BaseModel, Field
+
+
+class NoiseModelConfig(BaseModel):
+    """Configuration for noise model in simulated judgments.
+
+    Attributes
+    ----------
+    noise_type : Literal["temperature", "systematic", "random", "none"]
+        Type of noise to apply.
+    temperature : float
+        Temperature for scaling (higher = more random). Default: 1.0.
+    bias_strength : float
+        Strength of systematic biases (0.0-1.0). Default: 0.0.
+    bias_type : str | None
+        Type of bias ("length", "frequency", "position"). Default: None.
+    random_noise_stddev : float
+        Standard deviation for random noise. Default: 0.0.
+
+    Examples
+    --------
+    >>> # Temperature-scaled decisions (more random)
+    >>> config = NoiseModelConfig(noise_type="temperature", temperature=2.0)
+    >>>
+    >>> # Systematic length bias (prefer shorter)
+    >>> config = NoiseModelConfig(
+    ...     noise_type="systematic",
+    ...     bias_strength=0.3,
+    ...     bias_type="length"
+    ... )
+    >>>
+    >>> # Random noise injection
+    >>> config = NoiseModelConfig(
+    ...     noise_type="random",
+    ...     random_noise_stddev=0.1
+    ... )
+    """
+
+    noise_type: Literal["temperature", "systematic", "random", "none"] = Field(
+        default="temperature",
+        description="Type of noise model",
+    )
+    temperature: float = Field(
+        default=1.0,
+        ge=0.01,
+        le=10.0,
+        description="Temperature for scaling decisions",
+    )
+    bias_strength: float = Field(
+        default=0.0,
+        ge=0.0,
+        le=1.0,
+        description="Strength of systematic biases",
+    )
+    bias_type: str | None = Field(
+        default=None,
+        description="Type of systematic bias",
+    )
+    random_noise_stddev: float = Field(
+        default=0.0,
+        ge=0.0,
+        description="Standard deviation for random noise",
+    )
+
+
+class SimulatedAnnotatorConfig(BaseModel):
+    """Configuration for simulated annotator.
+
+    Attributes
+    ----------
+    strategy : Literal["lm_score", "distance", "random", "oracle", "dsl"]
+        Base strategy for generating judgments.
+    noise_model : NoiseModelConfig
+        Noise model configuration.
+    dsl_expression : str | None
+        Custom DSL expression for simulation logic.
+    random_state : int | None
+        Random seed for reproducibility.
+    model_output_key : str
+        Key to extract from Item.model_outputs. Default: "lm_score".
+    fallback_to_random : bool
+        Whether to fallback to random if model outputs missing. Default: True.
+
+    Examples
+    --------
+    >>> # LM score-based with temperature
+    >>> config = SimulatedAnnotatorConfig(
+    ...     strategy="lm_score",
+    ...     noise_model=NoiseModelConfig(noise_type="temperature", temperature=1.5),
+    ...     random_state=42
+    ... )
+    >>>
+    >>> # Distance-based with embeddings
+    >>> config = SimulatedAnnotatorConfig(
+    ...     strategy="distance",
+    ...     model_output_key="embedding",
+    ...     noise_model=NoiseModelConfig(noise_type="none")
+    ... )
+    >>>
+    >>> # Custom DSL logic
+    >>> config = SimulatedAnnotatorConfig(
+    ...     strategy="dsl",
+    ...     dsl_expression="sample_categorical(softmax(model_scores) / temperature)",
+    ...     noise_model=NoiseModelConfig(noise_type="temperature", temperature=1.0)
+    ... )
+    """
+
+    strategy: Literal["lm_score", "distance", "random", "oracle", "dsl"] = Field(
+        default="lm_score",
+        description="Base simulation strategy",
+    )
+    noise_model: NoiseModelConfig = Field(
+        default_factory=NoiseModelConfig,
+        description="Noise model configuration",
+    )
+    dsl_expression: str | None = Field(
+        default=None,
+        description="Custom DSL expression for simulation",
+    )
+    random_state: int | None = Field(
+        default=None,
+        description="Random seed for reproducibility",
+    )
+    model_output_key: str = Field(
+        default="lm_score",
+        description="Key to extract from model outputs",
+    )
+    fallback_to_random: bool = Field(
+        default=True,
+        description="Fallback to random if model outputs missing",
+    )
+
+
+class SimulationRunnerConfig(BaseModel):
+    """Configuration for simulation runner.
+
+    Attributes
+    ----------
+    annotator_configs : list[SimulatedAnnotatorConfig]
+        List of annotator configurations (for multi-annotator simulation).
+    n_annotators : int
+        Number of simulated annotators. Default: 1.
+    inter_annotator_correlation : float | None
+        Desired correlation between annotators (0.0-1.0). Default: None (independent).
+    output_format : Literal["dict", "dataframe", "jsonl"]
+        Output format for simulation results. Default: "dict".
+    save_path : Path | None
+        Path to save simulation results. Default: None.
+
+    Examples
+    --------
+    >>> # Single annotator
+    >>> config = SimulationRunnerConfig(
+    ...     annotator_configs=[SimulatedAnnotatorConfig(strategy="lm_score")],
+    ...     n_annotators=1
+    ... )
+    >>>
+    >>> # Multiple independent annotators
+    >>> config = SimulationRunnerConfig(
+    ...     annotator_configs=[
+    ...         SimulatedAnnotatorConfig(strategy="lm_score", random_state=1),
+    ...         SimulatedAnnotatorConfig(strategy="lm_score", random_state=2),
+    ...         SimulatedAnnotatorConfig(strategy="lm_score", random_state=3)
+    ...     ],
+    ...     n_annotators=3
+    ... )
+    >>>
+    >>> # Correlated annotators
+    >>> config = SimulationRunnerConfig(
+    ...     annotator_configs=[SimulatedAnnotatorConfig(strategy="lm_score")],
+    ...     n_annotators=5,
+    ...     inter_annotator_correlation=0.7  # 70% agreement
+    ... )
+    """
+
+    annotator_configs: list[SimulatedAnnotatorConfig] = Field(
+        default_factory=lambda: [SimulatedAnnotatorConfig()],
+        description="Annotator configurations",
+    )
+    n_annotators: int = Field(
+        default=1,
+        ge=1,
+        le=100,
+        description="Number of simulated annotators",
+    )
+    inter_annotator_correlation: float | None = Field(
+        default=None,
+        ge=0.0,
+        le=1.0,
+        description="Inter-annotator correlation",
+    )
+    output_format: Literal["dict", "dataframe", "jsonl"] = Field(
+        default="dict",
+        description="Output format",
+    )
+    save_path: Path | None = Field(
+        default=None,
+        description="Path to save results",
+    )

bead/config/template.py

@@ -0,0 +1,238 @@
+"""Template configuration models for the bead package."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Literal
+
+from pydantic import BaseModel, Field, field_validator, model_validator
+
+
+class SlotStrategyConfig(BaseModel):
+    """Configuration for a single slot's filling strategy.
+
+    Parameters
+    ----------
+    strategy
+        Filling strategy for this slot. Must be one of "exhaustive",
+        "random", "stratified", or "mlm".
+    sample_size
+        Sample size for random or stratified strategies. Only used when
+        strategy is "random" or "stratified".
+    stratify_by
+        Feature name to stratify by. Only used when strategy is "stratified".
+    beam_size
+        Beam size for MLM strategy. Only used when strategy is "mlm".
+
+    Examples
+    --------
+    >>> config = SlotStrategyConfig(strategy="exhaustive")
+    >>> config.strategy
+    'exhaustive'
+    >>> config_random = SlotStrategyConfig(strategy="random", sample_size=100)
+    >>> config_random.sample_size
+    100
+    >>> config_stratified = SlotStrategyConfig(
+    ...     strategy="stratified", sample_size=50, stratify_by="pos"
+    ... )
+    >>> config_stratified.stratify_by
+    'pos'
+    >>> config_mlm = SlotStrategyConfig(strategy="mlm", beam_size=10)
+    >>> config_mlm.beam_size
+    10
+    """
+
+    strategy: Literal["exhaustive", "random", "stratified", "mlm"] = Field(
+        ..., description="Filling strategy for this slot"
+    )
+    sample_size: int | None = Field(
+        default=None, description="Sample size for random/stratified"
+    )
+    stratify_by: str | None = Field(default=None, description="Feature to stratify by")
+    beam_size: int | None = Field(default=None, description="Beam size for MLM")
+
+
+class TemplateConfig(BaseModel):
+    """Configuration for template filling.
+
+    Parameters
+    ----------
+    filling_strategy : str
+        Strategy name for filling templates
+        ("exhaustive", "random", "stratified", "mlm", "mixed").
+    batch_size : int
+        Batch size for filling operations.
+    max_combinations : int | None
+        Maximum combinations to generate.
+    random_seed : int | None
+        Random seed for reproducibility.
+    stream_mode : bool
+        Use streaming for large templates.
+    use_csp_solver : bool
+        Use CSP solver for templates with multi-slot constraints.
+    mlm_model_name : str | None
+        HuggingFace model name for MLM filling.
+    mlm_beam_size : int
+        Beam search width for MLM strategy.
+    mlm_fill_direction : str
+        Direction for filling slots in MLM strategy.
+    mlm_custom_order : list[int] | None
+        Custom slot fill order for MLM strategy.
+    mlm_top_k : int
+        Number of top candidates per slot in MLM.
+    mlm_device : str
+        Device for MLM inference.
+    mlm_cache_enabled : bool
+        Enable content-addressable caching for MLM predictions.
+    mlm_cache_dir : Path | None
+        Directory for MLM prediction cache.
+    slot_strategies : dict[str, SlotStrategyConfig] | None
+        Per-slot strategy configuration for mixed filling.
+        Maps slot names to SlotStrategyConfig instances.
+
+    Examples
+    --------
+    >>> config = TemplateConfig()
+    >>> config.filling_strategy
+    'exhaustive'
+    >>> config.batch_size
+    1000
+    >>> # MLM configuration
+    >>> config_mlm = TemplateConfig(
+    ...     filling_strategy="mlm", mlm_model_name="bert-base-uncased"
+    ... )
+    >>> config_mlm.mlm_beam_size
+    5
+    >>> # Mixed strategy configuration
+    >>> config_mixed = TemplateConfig(
+    ...     filling_strategy="mixed",
+    ...     mlm_model_name="bert-base-uncased",
+    ...     slot_strategies={
+    ...         "noun": SlotStrategyConfig(strategy="exhaustive"),
+    ...         "verb": SlotStrategyConfig(strategy="exhaustive"),
+    ...         "adjective": SlotStrategyConfig(strategy="mlm", beam_size=10)
+    ...     }
+    ... )
+    >>> config_mixed.slot_strategies["noun"].strategy
+    'exhaustive'
+    >>> config_mixed.slot_strategies["adjective"].beam_size
+    10
+    """
+
+    filling_strategy: Literal["exhaustive", "random", "stratified", "mlm", "mixed"] = (
+        Field(default="exhaustive", description="Strategy for filling templates")
+    )
+    batch_size: int = Field(default=1000, description="Batch size for filling", gt=0)
+    max_combinations: int | None = Field(
+        default=None, description="Max combinations to generate"
+    )
+    random_seed: int | None = Field(
+        default=None, description="Random seed for reproducibility"
+    )
+    stream_mode: bool = Field(
+        default=False, description="Use streaming for large templates"
+    )
+    use_csp_solver: bool = Field(
+        default=False,
+        description="Use CSP solver for templates with multi-slot constraints",
+    )
+
+    # MLM-specific settings (model, beam size, fill direction)
+    mlm_model_name: str | None = Field(
+        default=None, description="HuggingFace model name for MLM filling"
+    )
+    mlm_beam_size: int = Field(
+        default=5, description="Beam search width for MLM strategy", gt=0
+    )
+    mlm_fill_direction: Literal[
+        "left_to_right", "right_to_left", "inside_out", "outside_in", "custom"
+    ] = Field(
+        default="left_to_right",
+        description="Direction for filling slots in MLM strategy",
+    )
+    mlm_custom_order: list[int] | None = Field(
+        default=None, description="Custom slot fill order for MLM strategy"
+    )
+    mlm_top_k: int = Field(
+        default=20, description="Number of top candidates per slot in MLM", gt=0
+    )
+    mlm_device: str = Field(default="cpu", description="Device for MLM inference")
+    mlm_cache_enabled: bool = Field(
+        default=True, description="Enable caching for MLM predictions"
+    )
+    mlm_cache_dir: Path | None = Field(
+        default=None, description="Directory for MLM prediction cache"
+    )
+
+    # mixed strategy settings
+    slot_strategies: dict[str, SlotStrategyConfig] | None = Field(
+        default=None,
+        description="Per-slot strategy configuration for mixed filling. "
+        "Maps slot names to SlotStrategyConfig instances.",
+    )
+
+    @field_validator("max_combinations")
+    @classmethod
+    def validate_max_combinations(cls, v: int | None) -> int | None:
+        """Validate max_combinations is positive.
+
+        Parameters
+        ----------
+        v : int | None
+            Max combinations value.
+
+        Returns
+        -------
+        int | None
+            Validated value.
+
+        Raises
+        ------
+        ValueError
+            If value is not positive.
+        """
+        if v is not None and v <= 0:
+            msg = f"max_combinations must be positive, got {v}"
+            raise ValueError(msg)
+        return v
+
+    @model_validator(mode="after")
+    def validate_mlm_config(self) -> TemplateConfig:
+        """Validate MLM configuration is consistent.
+
+        Returns
+        -------
+        TemplateConfig
+            Validated config.
+
+        Raises
+        ------
+        ValueError
+            If MLM config is inconsistent.
+        """
+        if self.filling_strategy == "mlm" and self.mlm_model_name is None:
+            msg = "mlm_model_name must be specified when filling_strategy is 'mlm'"
+            raise ValueError(msg)
+
+        if self.mlm_fill_direction == "custom" and self.mlm_custom_order is None:
+            msg = (
+                "mlm_custom_order must be specified when mlm_fill_direction is 'custom'"
+            )
+            raise ValueError(msg)
+
+        # validate mixed strategy configuration
+        if self.filling_strategy == "mixed" and self.slot_strategies is None:
+            msg = "slot_strategies must be specified when filling_strategy is 'mixed'"
+            raise ValueError(msg)
+
+        if self.slot_strategies is not None:
+            for slot_name, slot_config in self.slot_strategies.items():
+                # if MLM strategy is used for a slot, check model config is available
+                if slot_config.strategy == "mlm" and self.mlm_model_name is None:
+                    msg = (
+                        f"mlm_model_name must be specified when slot "
+                        f"'{slot_name}' uses MLM"
+                    )
+                    raise ValueError(msg)
+
+        return self

bead/config/validation.py

@@ -0,0 +1,267 @@
+"""Configuration validation utilities.
+
+This module provides pre-flight validation for configuration objects,
+checking for common issues before the configuration is used.
+"""
+
+from __future__ import annotations
+
+from bead.config.config import BeadConfig
+
+
+def check_paths_exist(config: BeadConfig) -> list[str]:
+    """Check that all configured paths exist or can be created.
+
+    Parameters
+    ----------
+    config : BeadConfig
+        Configuration to check.
+
+    Returns
+    -------
+    list[str]
+        List of path validation errors.
+
+    Examples
+    --------
+    >>> from bead.config import get_default_config
+    >>> config = get_default_config()
+    >>> errors = check_paths_exist(config)
+    >>> isinstance(errors, list)
+    True
+    """
+    errors: list[str] = []
+
+    # check main paths if they should exist and are absolute
+    if config.paths.data_dir.is_absolute() and not config.paths.data_dir.exists():
+        errors.append(f"data_dir does not exist: {config.paths.data_dir}")
+
+    if config.paths.output_dir.is_absolute() and not config.paths.output_dir.exists():
+        errors.append(f"output_dir does not exist: {config.paths.output_dir}")
+
+    if config.paths.cache_dir.is_absolute() and not config.paths.cache_dir.exists():
+        errors.append(f"cache_dir does not exist: {config.paths.cache_dir}")
+
+    # check resource paths
+    if (
+        config.resources.lexicon_path is not None
+        and config.resources.lexicon_path.is_absolute()
+        and not config.resources.lexicon_path.exists()
+    ):
+        errors.append(f"lexicon_path does not exist: {config.resources.lexicon_path}")
+
+    if (
+        config.resources.templates_path is not None
+        and config.resources.templates_path.is_absolute()
+        and not config.resources.templates_path.exists()
+    ):
+        errors.append(
+            f"templates_path does not exist: {config.resources.templates_path}"
+        )
+
+    if (
+        config.resources.constraints_path is not None
+        and config.resources.constraints_path.is_absolute()
+        and not config.resources.constraints_path.exists()
+    ):
+        errors.append(
+            f"constraints_path does not exist: {config.resources.constraints_path}"
+        )
+
+    # check training logging dir
+    if (
+        config.active_learning.trainer.logging_dir.is_absolute()
+        and not config.active_learning.trainer.logging_dir.exists()
+    ):
+        errors.append(
+            f"logging_dir does not exist: {config.active_learning.trainer.logging_dir}"
+        )
+
+    # check logging file parent directory
+    if (
+        config.logging.file is not None
+        and config.logging.file.is_absolute()
+        and not config.logging.file.parent.exists()
+    ):
+        parent_dir = config.logging.file.parent
+        errors.append(f"logging file parent directory does not exist: {parent_dir}")
+
+    return errors
+
+
+def check_resource_compatibility(config: BeadConfig) -> list[str]:
+    """Verify resources are compatible with templates.
+
+    Parameters
+    ----------
+    config : BeadConfig
+        Configuration to check.
+
+    Returns
+    -------
+    list[str]
+        List of resource compatibility errors.
+    """
+    errors: list[str] = []
+
+    # check that if templates_path is specified, lexicon_path should also be specified
+    if (
+        config.resources.templates_path is not None
+        and config.resources.lexicon_path is None
+    ):
+        errors.append(
+            "templates_path is specified but lexicon_path is not. "
+            "Templates require a lexicon."
+        )
+
+    return errors
+
+
+def check_model_configuration(config: BeadConfig) -> list[str]:
+    """Verify model settings are valid.
+
+    Parameters
+    ----------
+    config : BeadConfig
+        Configuration to check.
+
+    Returns
+    -------
+    list[str]
+        List of model configuration errors.
+    """
+    try:
+        import torch  # noqa: PLC0415
+    except ImportError:
+        torch = None  # type: ignore[assignment]
+
+    errors: list[str] = []
+
+    # check CUDA availability if device is set to cuda
+    if config.items.model.device == "cuda":
+        if torch is None:
+            errors.append(
+                "Model device is set to 'cuda' but PyTorch is not installed. "
+                "Install PyTorch or set device to 'cpu'."
+            )
+        elif not torch.cuda.is_available():  # type: ignore[no-untyped-call]
+            errors.append(
+                "Model device is set to 'cuda' but CUDA is not available. "
+                "Set device to 'cpu' or install CUDA."
+            )
+
+    # check MPS availability if device is set to mps
+    if config.items.model.device == "mps":
+        if torch is None:
+            errors.append(
+                "Model device is set to 'mps' but PyTorch is not installed. "
+                "Install PyTorch or set device to 'cpu'."
+            )
+        elif not torch.backends.mps.is_available():  # type: ignore[no-untyped-call]
+            errors.append(
+                "Model device is set to 'mps' but MPS is not available. "
+                "Set device to 'cpu' or use a macOS system with MPS support."
+            )
+
+    return errors
+
+
+def check_training_configuration(config: BeadConfig) -> list[str]:
+    """Verify training settings are compatible.
+
+    Parameters
+    ----------
+    config : BeadConfig
+        Configuration to check.
+
+    Returns
+    -------
+    list[str]
+        List of training configuration errors.
+    """
+    errors: list[str] = []
+
+    # check that batch size is positive
+    if config.active_learning.forced_choice_model.batch_size <= 0:
+        batch_size = config.active_learning.forced_choice_model.batch_size
+        errors.append(f"Training batch size must be positive, got {batch_size}")
+
+    # check that epochs is positive
+    if config.active_learning.trainer.epochs <= 0:
+        epochs = config.active_learning.trainer.epochs
+        errors.append(f"Training epochs must be positive, got {epochs}")
+
+    # check that learning rate is positive
+    if config.active_learning.forced_choice_model.learning_rate <= 0:
+        lr = config.active_learning.forced_choice_model.learning_rate
+        errors.append(f"Training learning rate must be positive, got {lr}")
+
+    return errors
+
+
+def check_deployment_configuration(config: BeadConfig) -> list[str]:
+    """Verify deployment settings are valid.
+
+    Parameters
+    ----------
+    config : BeadConfig
+        Configuration to check.
+
+    Returns
+    -------
+    list[str]
+        List of deployment configuration errors.
+    """
+    errors: list[str] = []
+
+    # check jsPsych version format if platform is jspsych
+    if config.deployment.platform == "jspsych":
+        version = config.deployment.jspsych_version
+        if version is None:  # type: ignore[reportUnnecessaryComparison]
+            errors.append("jsPsych platform requires jspsych_version to be specified")
+        elif not isinstance(version, str):  # type: ignore[reportUnnecessaryIsInstance]
+            errors.append(
+                f"jspsych_version must be a string, got {type(version).__name__}"
+            )
+
+    return errors
+
+
+def validate_config(config: BeadConfig) -> list[str]:
+    """Perform pre-flight validation on configuration.
+
+    Checks:
+    - All paths exist (if absolute paths are specified)
+    - Resource paths exist (if specified)
+    - Model configurations are compatible
+    - Training configurations are valid
+    - No conflicting settings
+
+    Parameters
+    ----------
+    config : BeadConfig
+        Configuration to validate.
+
+    Returns
+    -------
+    list[str]
+        List of validation errors. Empty if valid.
+
+    Examples
+    --------
+    >>> from bead.config import get_default_config
+    >>> config = get_default_config()
+    >>> errors = validate_config(config)
+    >>> len(errors)
+    0
+    """
+    errors: list[str] = []
+
+    # run all validation checks
+    errors.extend(check_paths_exist(config))
+    errors.extend(check_resource_compatibility(config))
+    errors.extend(check_model_configuration(config))
+    errors.extend(check_training_configuration(config))
+    errors.extend(check_deployment_configuration(config))
+
+    return errors