themis-eval 0.1.0__py3-none-any.whl → 0.2.0__py3-none-any.whl

themis/comparison/statistics.py

@@ -0,0 +1,402 @@
+"""Statistical tests for comparing experiment results.
+
+This module provides various statistical tests to determine if differences
+between runs are statistically significant.
+"""
+
+from __future__ import annotations
+
+import math
+import random
+from dataclasses import dataclass
+from enum import Enum
+from typing import Sequence
+
+
+class StatisticalTest(str, Enum):
+    """Available statistical tests."""
+    
+    T_TEST = "t_test"
+    BOOTSTRAP = "bootstrap"
+    PERMUTATION = "permutation"
+    NONE = "none"
+
+
+@dataclass
+class StatisticalTestResult:
+    """Result of a statistical test.
+    
+    Attributes:
+        test_name: Name of the test performed
+        statistic: Test statistic value
+        p_value: P-value (probability of observing this difference by chance)
+        significant: Whether the difference is statistically significant
+        confidence_level: Confidence level used (e.g., 0.95 for 95%)
+        effect_size: Effect size (e.g., Cohen's d)
+        confidence_interval: Confidence interval for the difference
+    """
+    
+    test_name: str
+    statistic: float
+    p_value: float
+    significant: bool
+    confidence_level: float = 0.95
+    effect_size: float | None = None
+    confidence_interval: tuple[float, float] | None = None
+    
+    def __str__(self) -> str:
+        """Human-readable summary."""
+        sig_str = "significant" if self.significant else "not significant"
+        result = f"{self.test_name}: p={self.p_value:.4f} ({sig_str})"
+        
+        if self.effect_size is not None:
+            result += f", effect_size={self.effect_size:.3f}"
+        
+        if self.confidence_interval is not None:
+            low, high = self.confidence_interval
+            result += f", CI=[{low:.3f}, {high:.3f}]"
+        
+        return result
+
+
+def t_test(
+    samples_a: Sequence[float],
+    samples_b: Sequence[float],
+    *,
+    alpha: float = 0.05,
+    paired: bool = True,
+) -> StatisticalTestResult:
+    """Perform a t-test to compare two sets of samples.
+    
+    Args:
+        samples_a: First set of samples
+        samples_b: Second set of samples
+        alpha: Significance level (default: 0.05 for 95% confidence)
+        paired: Whether to use paired t-test (default: True)
+    
+    Returns:
+        StatisticalTestResult with test statistics and significance
+    
+    Raises:
+        ValueError: If samples are empty or have mismatched lengths (for paired test)
+    """
+    if not samples_a or not samples_b:
+        raise ValueError("Cannot perform t-test on empty samples")
+    
+    if paired and len(samples_a) != len(samples_b):
+        raise ValueError(
+            f"Paired t-test requires equal sample sizes. "
+            f"Got {len(samples_a)} and {len(samples_b)}"
+        )
+    
+    n_a = len(samples_a)
+    n_b = len(samples_b)
+    
+    # Calculate means
+    mean_a = sum(samples_a) / n_a
+    mean_b = sum(samples_b) / n_b
+    
+    if paired:
+        # Paired t-test: test on differences
+        diffs = [a - b for a, b in zip(samples_a, samples_b)]
+        mean_diff = sum(diffs) / len(diffs)
+        
+        # Standard deviation of differences
+        var_diff = sum((d - mean_diff) ** 2 for d in diffs) / (len(diffs) - 1) if len(diffs) > 1 else 0
+        se_diff = math.sqrt(var_diff / len(diffs))
+        
+        # T-statistic
+        if se_diff > 1e-10:  # Non-zero standard error
+            t_stat = mean_diff / se_diff
+        elif abs(mean_diff) > 1e-10:  # Perfect consistency with non-zero difference
+            t_stat = float('inf') if mean_diff > 0 else float('-inf')
+        else:  # No difference at all
+            t_stat = 0.0
+        
+        df = len(diffs) - 1
+        
+        # Effect size (Cohen's d for paired samples)
+        sd_diff = math.sqrt(var_diff)
+        effect_size = mean_diff / sd_diff if sd_diff > 1e-10 else (1.0 if abs(mean_diff) > 1e-10 else 0.0)
+        
+    else:
+        # Independent samples t-test
+        # Calculate pooled standard deviation
+        var_a = sum((x - mean_a) ** 2 for x in samples_a) / (n_a - 1) if n_a > 1 else 0
+        var_b = sum((x - mean_b) ** 2 for x in samples_b) / (n_b - 1) if n_b > 1 else 0
+        
+        pooled_sd = math.sqrt(((n_a - 1) * var_a + (n_b - 1) * var_b) / (n_a + n_b - 2))
+        se = pooled_sd * math.sqrt(1/n_a + 1/n_b)
+        
+        # T-statistic
+        t_stat = (mean_a - mean_b) / se if se > 0 else 0.0
+        df = n_a + n_b - 2
+        
+        # Effect size (Cohen's d)
+        effect_size = (mean_a - mean_b) / pooled_sd if pooled_sd > 0 else 0.0
+    
+    # Approximate p-value using t-distribution
+    # For simplicity, we use a conservative approximation
+    # In practice, you'd use scipy.stats.t.sf for accurate p-values
+    p_value = _approximate_t_test_p_value(abs(t_stat), df)
+    
+    # Confidence interval (approximate)
+    # t_critical ≈ 2.0 for 95% CI and reasonable df
+    t_critical = 2.0  # Conservative estimate
+    margin = t_critical * (se_diff if paired else se)
+    ci = (mean_a - mean_b - margin, mean_a - mean_b + margin)
+    
+    return StatisticalTestResult(
+        test_name="t-test (paired)" if paired else "t-test (independent)",
+        statistic=t_stat,
+        p_value=p_value,
+        significant=p_value < alpha,
+        confidence_level=1 - alpha,
+        effect_size=effect_size,
+        confidence_interval=ci,
+    )
+
+
+def _approximate_t_test_p_value(t_stat: float, df: int) -> float:
+    """Approximate p-value for t-test.
+    
+    This is a rough approximation. For accurate p-values, use scipy.stats.
+    """
+    # Very rough approximation based on standard normal
+    # This gets less accurate for small df
+    if df < 1:
+        return 1.0
+    
+    # Convert to z-score approximation for large df
+    if df > 30:
+        z = t_stat
+        # Approximate p-value for two-tailed test
+        # P(|Z| > z) ≈ 2 * (1 - Φ(z))
+        if z > 6:
+            return 0.0
+        elif z < 0.5:
+            return 1.0
+        else:
+            # Rough approximation
+            return min(1.0, 2 * math.exp(-0.5 * z * z) / math.sqrt(2 * math.pi))
+    
+    # For small df, be conservative
+    return min(1.0, 0.5 if t_stat < 2 else 0.1 if t_stat < 3 else 0.01)
+
+
+def bootstrap_confidence_interval(
+    samples_a: Sequence[float],
+    samples_b: Sequence[float],
+    *,
+    n_bootstrap: int = 10000,
+    confidence_level: float = 0.95,
+    statistic_fn: callable = None,
+    seed: int | None = None,
+) -> StatisticalTestResult:
+    """Compute bootstrap confidence interval for difference between two samples.
+    
+    Uses bootstrap resampling to estimate the confidence interval for the
+    difference in means (or other statistic) between two samples.
+    
+    Args:
+        samples_a: First set of samples
+        samples_b: Second set of samples
+        n_bootstrap: Number of bootstrap iterations (default: 10000)
+        confidence_level: Confidence level (default: 0.95)
+        statistic_fn: Function to compute statistic (default: mean difference)
+        seed: Random seed for reproducibility
+    
+    Returns:
+        StatisticalTestResult with bootstrap confidence interval
+    """
+    if not samples_a or not samples_b:
+        raise ValueError("Cannot perform bootstrap on empty samples")
+    
+    if seed is not None:
+        random.seed(seed)
+    
+    # Default statistic: difference in means
+    if statistic_fn is None:
+        def statistic_fn(a, b):
+            return sum(a) / len(a) - sum(b) / len(b)
+    
+    # Observed difference
+    observed_diff = statistic_fn(samples_a, samples_b)
+    
+    # Bootstrap resampling
+    bootstrap_diffs = []
+    for _ in range(n_bootstrap):
+        # Resample with replacement
+        resampled_a = [random.choice(samples_a) for _ in range(len(samples_a))]
+        resampled_b = [random.choice(samples_b) for _ in range(len(samples_b))]
+        
+        diff = statistic_fn(resampled_a, resampled_b)
+        bootstrap_diffs.append(diff)
+    
+    # Sort for percentile method
+    bootstrap_diffs.sort()
+    
+    # Compute confidence interval
+    alpha = 1 - confidence_level
+    lower_idx = int(n_bootstrap * (alpha / 2))
+    upper_idx = int(n_bootstrap * (1 - alpha / 2))
+    
+    ci = (bootstrap_diffs[lower_idx], bootstrap_diffs[upper_idx])
+    
+    # Check if 0 is in the confidence interval
+    significant = not (ci[0] <= 0 <= ci[1])
+    
+    # Pseudo p-value: proportion of bootstrap samples with opposite sign
+    p_value = sum(1 for d in bootstrap_diffs if (d * observed_diff) < 0) / n_bootstrap
+    p_value = max(p_value, 1 / n_bootstrap)  # Lower bound
+    
+    return StatisticalTestResult(
+        test_name=f"bootstrap (n={n_bootstrap})",
+        statistic=observed_diff,
+        p_value=p_value,
+        significant=significant,
+        confidence_level=confidence_level,
+        confidence_interval=ci,
+    )
+
+
+def permutation_test(
+    samples_a: Sequence[float],
+    samples_b: Sequence[float],
+    *,
+    n_permutations: int = 10000,
+    alpha: float = 0.05,
+    statistic_fn: callable = None,
+    seed: int | None = None,
+) -> StatisticalTestResult:
+    """Perform permutation test to compare two samples.
+    
+    Tests the null hypothesis that the two samples come from the same
+    distribution by randomly permuting the labels and computing the test
+    statistic.
+    
+    Args:
+        samples_a: First set of samples
+        samples_b: Second set of samples
+        n_permutations: Number of permutations (default: 10000)
+        alpha: Significance level (default: 0.05)
+        statistic_fn: Function to compute statistic (default: difference in means)
+        seed: Random seed for reproducibility
+    
+    Returns:
+        StatisticalTestResult with permutation test results
+    """
+    if not samples_a or not samples_b:
+        raise ValueError("Cannot perform permutation test on empty samples")
+    
+    if seed is not None:
+        random.seed(seed)
+    
+    # Default statistic: absolute difference in means
+    if statistic_fn is None:
+        def statistic_fn(a, b):
+            return abs(sum(a) / len(a) - sum(b) / len(b))
+    
+    # Observed statistic
+    observed_stat = statistic_fn(samples_a, samples_b)
+    
+    # Combine all samples
+    combined = list(samples_a) + list(samples_b)
+    n_a = len(samples_a)
+    n_total = len(combined)
+    
+    # Permutation testing
+    more_extreme = 0
+    for _ in range(n_permutations):
+        # Shuffle and split
+        shuffled = combined.copy()
+        random.shuffle(shuffled)
+        
+        perm_a = shuffled[:n_a]
+        perm_b = shuffled[n_a:]
+        
+        perm_stat = statistic_fn(perm_a, perm_b)
+        
+        if perm_stat >= observed_stat:
+            more_extreme += 1
+    
+    # P-value: proportion of permutations as extreme as observed
+    p_value = more_extreme / n_permutations
+    
+    return StatisticalTestResult(
+        test_name=f"permutation (n={n_permutations})",
+        statistic=observed_stat,
+        p_value=p_value,
+        significant=p_value < alpha,
+        confidence_level=1 - alpha,
+    )
+
+
+def mcnemar_test(
+    contingency_table: tuple[int, int, int, int],
+    *,
+    alpha: float = 0.05,
+) -> StatisticalTestResult:
+    """Perform McNemar's test for paired nominal data.
+    
+    Useful for comparing two models on the same test set, where you want to
+    know if one model consistently outperforms the other.
+    
+    Args:
+        contingency_table: 2x2 contingency table as (n_00, n_01, n_10, n_11)
+            where n_ij = number of samples where model A predicts i and model B predicts j
+            (0 = incorrect, 1 = correct)
+        alpha: Significance level
+    
+    Returns:
+        StatisticalTestResult with McNemar's test results
+    """
+    n_00, n_01, n_10, n_11 = contingency_table
+    
+    # Only discordant pairs matter
+    b = n_01  # A wrong, B correct
+    c = n_10  # A correct, B wrong
+    
+    if b + c == 0:
+        # No discordant pairs
+        return StatisticalTestResult(
+            test_name="McNemar's test",
+            statistic=0.0,
+            p_value=1.0,
+            significant=False,
+            confidence_level=1 - alpha,
+        )
+    
+    # McNemar's statistic with continuity correction
+    chi_square = ((abs(b - c) - 1) ** 2) / (b + c)
+    
+    # Approximate p-value (chi-square with 1 df)
+    # For chi-square > 3.84, p < 0.05
+    # For chi-square > 6.63, p < 0.01
+    if chi_square > 10.83:
+        p_value = 0.001
+    elif chi_square > 6.63:
+        p_value = 0.01
+    elif chi_square > 3.84:
+        p_value = 0.05
+    else:
+        # Rough linear approximation
+        p_value = 1.0 - (chi_square / 3.84) * 0.95
+    
+    return StatisticalTestResult(
+        test_name="McNemar's test",
+        statistic=chi_square,
+        p_value=p_value,
+        significant=p_value < alpha,
+        confidence_level=1 - alpha,
+    )
+
+
+__all__ = [
+    "StatisticalTest",
+    "StatisticalTestResult",
+    "t_test",
+    "bootstrap_confidence_interval",
+    "permutation_test",
+    "mcnemar_test",
+]

themis/config/__init__.py

@@ -0,0 +1,19 @@
+"""Hydra-backed configuration helpers for assembling experiments."""
+
+from __future__ import annotations
+
+from .loader import load_experiment_config
+from .runtime import (
+    load_dataset_from_config,
+    run_experiment_from_config,
+    summarize_report_for_config,
+)
+from .schema import ExperimentConfig
+
+__all__ = [
+    "ExperimentConfig",
+    "load_dataset_from_config",
+    "load_experiment_config",
+    "run_experiment_from_config",
+    "summarize_report_for_config",
+]

themis/config/loader.py

@@ -0,0 +1,27 @@
+"""Utilities for loading experiment configs via Hydra/OmegaConf."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Iterable
+
+from omegaconf import OmegaConf
+
+from . import schema
+
+
+def load_experiment_config(
+    config_path: Path,
+    overrides: Iterable[str] | None = None,
+) -> schema.ExperimentConfig:
+    """Load and validate an experiment config file with optional overrides."""
+
+    base = OmegaConf.structured(schema.ExperimentConfig)
+    file_conf = OmegaConf.load(config_path)
+    merged = OmegaConf.merge(base, file_conf)
+
+    if overrides:
+        override_conf = OmegaConf.from_dotlist(list(overrides))
+        merged = OmegaConf.merge(merged, override_conf)
+
+    return OmegaConf.to_object(merged)

themis/config/registry.py

@@ -0,0 +1,34 @@
+"""Registry for experiment builders."""
+
+from __future__ import annotations
+
+from typing import Callable
+
+from themis.config import schema
+from themis.experiment import orchestrator
+
+ExperimentBuilder = Callable[
+    [schema.ExperimentConfig], orchestrator.ExperimentOrchestrator
+]
+
+_EXPERIMENT_BUILDERS: dict[str, ExperimentBuilder] = {}
+
+
+def register_experiment_builder(task: str) -> Callable[[ExperimentBuilder], ExperimentBuilder]:
+    """Decorator to register an experiment builder for a specific task."""
+
+    def decorator(builder: ExperimentBuilder) -> ExperimentBuilder:
+        _EXPERIMENT_BUILDERS[task] = builder
+        return builder
+
+    return decorator
+
+
+def get_experiment_builder(task: str) -> ExperimentBuilder:
+    """Get the experiment builder for a specific task."""
+    if task not in _EXPERIMENT_BUILDERS:
+        raise ValueError(
+            f"No experiment builder registered for task '{task}'. "
+            f"Available tasks: {', '.join(sorted(_EXPERIMENT_BUILDERS.keys()))}"
+        )
+    return _EXPERIMENT_BUILDERS[task]

themis/config/runtime.py

@@ -0,0 +1,214 @@
+"""Runtime helpers for executing experiments from Hydra configs."""
+
+from __future__ import annotations
+
+from dataclasses import asdict
+from pathlib import Path
+from typing import List
+
+from themis.core import entities as core_entities
+from themis.datasets import create_dataset
+from themis.experiment import math as math_experiment
+from themis.experiment import mcq as mcq_experiment
+from themis.experiment import orchestrator as experiment_orchestrator
+from themis.experiment import storage as experiment_storage
+from themis.providers import registry as provider_registry
+
+from . import registry, schema
+
+
+
+
+def run_experiment_from_config(
+    config: schema.ExperimentConfig,
+    *,
+    dataset: list[dict[str, object]] | None = None,
+    on_result=None,
+) -> experiment_orchestrator.ExperimentReport:
+    dataset_to_use = (
+        dataset
+        if dataset is not None
+        else _load_dataset(config.dataset, experiment_name=config.name)
+    )
+    experiment = _build_experiment(config)
+    return experiment.run(
+        dataset_to_use,
+        max_samples=config.max_samples,
+        run_id=config.run_id,
+        resume=config.resume,
+        on_result=on_result,
+    )
+
+
+def summarize_report_for_config(
+    config: schema.ExperimentConfig,
+    report: experiment_orchestrator.ExperimentReport,
+) -> str:
+    if config.task in {
+        "math500",
+        "aime24",
+        "aime25",
+        "amc23",
+        "olympiadbench",
+        "beyondaime",
+    }:
+        return math_experiment.summarize_report(report)
+    if config.task in {"supergpqa", "mmlu_pro"}:
+        return mcq_experiment.summarize_report(report)
+    raise ValueError(f"Unsupported task '{config.task}' for summarization.")
+
+
+def load_dataset_from_config(
+    config: schema.ExperimentConfig,
+) -> list[dict[str, object]]:
+    return _load_dataset(config.dataset, experiment_name=config.name)
+
+
+def _build_experiment(
+    config: schema.ExperimentConfig,
+) -> experiment_orchestrator.ExperimentOrchestrator:
+    if config.task:
+        builder = registry.get_experiment_builder(config.task)
+        return builder(config)
+
+    raise ValueError(
+        "Experiment configuration must specify a 'task'. "
+        f"Available tasks: {', '.join(sorted(registry._EXPERIMENT_BUILDERS.keys()))}"
+    )
+
+
+@registry.register_experiment_builder("math500")
+@registry.register_experiment_builder("aime24")
+@registry.register_experiment_builder("aime25")
+@registry.register_experiment_builder("amc23")
+@registry.register_experiment_builder("olympiadbench")
+@registry.register_experiment_builder("beyondaime")
+def _build_math_experiment(
+    config: schema.ExperimentConfig,
+) -> experiment_orchestrator.ExperimentOrchestrator:
+    # Use the specific path if provided, otherwise use the default path
+    storage_path = config.storage.path or config.storage.default_path
+    storage = (
+        experiment_storage.ExperimentStorage(Path(storage_path))
+        if storage_path
+        else None
+    )
+    sampling_cfg = core_entities.SamplingConfig(
+        temperature=config.generation.sampling.temperature,
+        top_p=config.generation.sampling.top_p,
+        max_tokens=config.generation.sampling.max_tokens,
+    )
+    provider = provider_registry.create_provider(
+        config.generation.provider.name, **config.generation.provider.options
+    )
+    runner_options = asdict(config.generation.runner)
+
+    # Use the task name from config as the default task name
+    task_name = config.task or "math500"
+    # Override task name if provided in task_options
+    if config.task_options and "task_name" in config.task_options:
+        task_name = config.task_options["task_name"]
+
+    return math_experiment.build_math500_zero_shot_experiment(
+        model_client=provider,
+        model_name=config.generation.model_identifier,
+        storage=storage,
+        sampling=sampling_cfg,
+        provider_name=config.generation.provider.name,
+        runner_options=runner_options,
+        task_name=task_name,
+    )
+
+
+@registry.register_experiment_builder("supergpqa")
+def _build_supergpqa_experiment(
+    config: schema.ExperimentConfig,
+) -> experiment_orchestrator.ExperimentOrchestrator:
+    return _build_mcq_experiment(config, "supergpqa", "supergpqa")
+
+
+@registry.register_experiment_builder("mmlu_pro")
+def _build_mmlu_pro_experiment(
+    config: schema.ExperimentConfig,
+) -> experiment_orchestrator.ExperimentOrchestrator:
+    return _build_mcq_experiment(config, "mmlu-pro", "mmlu_pro")
+
+
+def _build_mcq_experiment(
+    config: schema.ExperimentConfig, dataset_name: str, task_id: str
+) -> experiment_orchestrator.ExperimentOrchestrator:
+    # Use the specific path if provided, otherwise use the default path
+    storage_path = config.storage.path or config.storage.default_path
+    storage = (
+        experiment_storage.ExperimentStorage(Path(storage_path))
+        if storage_path
+        else None
+    )
+    sampling_cfg = core_entities.SamplingConfig(
+        temperature=config.generation.sampling.temperature,
+        top_p=config.generation.sampling.top_p,
+        max_tokens=config.generation.sampling.max_tokens,
+    )
+    provider = provider_registry.create_provider(
+        config.generation.provider.name, **config.generation.provider.options
+    )
+    runner_options = asdict(config.generation.runner)
+
+    return mcq_experiment.build_multiple_choice_json_experiment(
+        dataset_name=dataset_name,
+        task_id=task_id,
+        model_client=provider,
+        model_name=config.generation.model_identifier,
+        storage=storage,
+        sampling=sampling_cfg,
+        provider_name=config.generation.provider.name,
+        runner_options=runner_options,
+    )
+
+
+def _load_dataset(
+    config: schema.DatasetConfig, *, experiment_name: str
+) -> List[dict[str, object]]:
+    """Load dataset samples using the dataset registry.
+
+    Args:
+        config: Dataset configuration
+        experiment_name: Name of the experiment (used to map to dataset)
+
+    Returns:
+        List of sample dictionaries ready for generation
+    """
+    # Handle inline datasets (not in registry)
+    if config.source == "inline":
+        if not config.inline_samples:
+            raise ValueError(
+                "dataset.inline_samples must contain at least one row when"
+                " dataset.source='inline'."
+            )
+        return list(config.inline_samples)
+
+    # Use explicit dataset_id if provided
+    dataset_name = config.dataset_id
+    if not dataset_name:
+        # Fallback to task name if dataset_id is not provided
+        # This allows simple configs where task name matches dataset name
+        # But we should probably enforce dataset_id for clarity in the future
+        # For now, let's try to infer from task if available in config object passed to this function?
+        # Wait, _load_dataset only gets DatasetConfig and experiment_name.
+        # We should probably pass the full config or at least the task.
+        # But for now, let's rely on dataset_id being present or raise error.
+        raise ValueError(
+            "dataset.dataset_id must be provided when source is not 'inline'."
+        )
+
+    # Prepare options for dataset factory
+    options = {
+        "source": config.source,
+        "data_dir": config.data_dir,
+        "split": config.split,
+        "limit": config.limit,
+        "subjects": list(config.subjects) if config.subjects else None,
+    }
+
+    # Load samples via registry
+    return create_dataset(dataset_name, **options)