themis-eval 0.1.0__py3-none-any.whl → 0.2.0__py3-none-any.whl

themis/evaluation/pipelines/standard_pipeline.py

@@ -0,0 +1,348 @@
+"""Standard evaluation pipeline implementation."""
+
+from __future__ import annotations
+
+import logging
+import time
+import warnings
+from typing import Callable, Sequence
+
+from themis.core import entities as core_entities
+from themis.evaluation import extractors
+from themis.evaluation import strategies as evaluation_strategies
+from themis.evaluation.reports import (
+    EvaluationFailure,
+    EvaluationReport,
+    MetricAggregate,
+)
+from themis.interfaces import Metric as MetricInterface
+from themis.utils import tracing
+
+logger = logging.getLogger(__name__)
+
+
+def _default_reference_selector(record: core_entities.GenerationRecord):
+    """Default reference selector from generation record.
+
+    Args:
+        record: Generation record
+
+    Returns:
+        Reference value or None
+    """
+    reference = record.task.reference
+    if reference is None:
+        return None
+    return reference.value
+
+
+def _normalize_references(reference) -> list:
+    """Normalize reference to list format for metric consumption.
+
+    This function converts various reference formats into a standardized list
+    that metrics can reliably consume. The normalized format is always a list
+    where each element represents one reference value.
+
+    Args:
+        reference: Reference value in various formats:
+            - Reference object: Extracts .value field
+            - dict: Kept as-is in a list (for multi-value references)
+            - list/tuple: Returned as list
+            - scalar: Wrapped in a list
+
+    Returns:
+        List of reference values. Each element can be:
+        - A scalar value (str, int, float, bool)
+        - A dict (for multi-value references like {"target": 122, "numbers": [...]})
+        - Any other type from the original reference
+
+    Examples:
+        >>> _normalize_references(Reference(kind="answer", value="42"))
+        ["42"]
+
+        >>> _normalize_references(Reference(kind="task", value={"target": 122, "numbers": [25, 50]}))
+        [{"target": 122, "numbers": [25, 50]}]
+
+        >>> _normalize_references(["yes", "no", "maybe"])
+        ["yes", "no", "maybe"]
+
+        >>> _normalize_references("42")
+        ["42"]
+
+    Note:
+        Metrics receive references in this normalized format and should handle
+        both simple values and dict values appropriately.
+    """
+    if isinstance(reference, core_entities.Reference):
+        reference = reference.value
+    if isinstance(reference, list):
+        return reference
+    if isinstance(reference, tuple):
+        return list(reference)
+    return [reference]
+
+
+class EvaluationPipeline:
+    """Traditional batch evaluation pipeline.
+
+    This pipeline evaluates generation records using extractors, metrics,
+    and evaluation strategies. It supports slicing for subset analysis.
+
+    Example:
+        >>> pipeline = EvaluationPipeline(
+        ...     extractor=JsonFieldExtractor("answer"),
+        ...     metrics=[ExactMatch()]
+        ... )
+        >>> report = pipeline.evaluate(records)
+
+    Attributes:
+        _extractor: Extractor for parsing model output
+        _metrics: List of metrics to compute
+        _reference_selector: Function to extract reference from record
+        _strategy_resolver: Function to resolve evaluation strategy
+        _slices: List of (name, predicate) tuples for slicing
+    """
+
+    def __init__(
+        self,
+        *,
+        extractor,
+        metrics: Sequence[MetricInterface],
+        reference_selector: Callable[[core_entities.GenerationRecord], object]
+        | None = None,
+        strategy_resolver: Callable[
+            [core_entities.GenerationRecord], evaluation_strategies.EvaluationStrategy
+        ]
+        | None = None,
+    ) -> None:
+        """Initialize evaluation pipeline.
+
+        Args:
+            extractor: Extractor for parsing model output
+            metrics: List of metrics to compute
+            reference_selector: Optional function to extract reference from record.
+                If provided, this takes precedence over item.reference from strategies.
+            strategy_resolver: Optional function to resolve evaluation strategy.
+                If using a custom reference_selector with DefaultEvaluationStrategy,
+                the selector will take precedence.
+
+        Note:
+            When using DefaultEvaluationStrategy with a custom reference_selector,
+            the reference_selector will override the default behavior. Consider
+            using a custom strategy if you need more control over reference selection.
+        """
+        self._extractor = extractor
+        self._metrics = list(metrics)
+        self._reference_selector = reference_selector
+        self._has_custom_reference_selector = reference_selector is not None
+        self._strategy_resolver = strategy_resolver or (
+            lambda record: evaluation_strategies.DefaultEvaluationStrategy()
+        )
+        self._slices: list[
+            tuple[str, Callable[[core_entities.GenerationRecord], bool]]
+        ] = []
+
+        # Validation: warn if custom reference_selector is used with default strategy
+        if self._has_custom_reference_selector and strategy_resolver is None:
+            warnings.warn(
+                "Custom reference_selector provided without custom strategy_resolver. "
+                "The reference_selector will take precedence over DefaultEvaluationStrategy's "
+                "reference handling. If you need more control, consider providing a custom "
+                "strategy_resolver that sets reference=None in EvaluationItem.",
+                UserWarning,
+                stacklevel=2,
+            )
+
+    def evaluate(
+        self, records: Sequence[core_entities.GenerationRecord]
+    ) -> EvaluationReport:
+        """Evaluate generation records.
+
+        Args:
+            records: Generation records to evaluate
+
+        Returns:
+            Evaluation report with metrics and failures
+        """
+        with tracing.span("evaluate_pipeline", total_records=len(records)):
+            per_metric: dict[str, list[core_entities.MetricScore]] = {
+                metric.name: [] for metric in self._metrics
+            }
+            failures: list[EvaluationFailure] = []
+            per_record: list[core_entities.EvaluationRecord] = []
+            slice_members: dict[str, set[str]] = {
+                name: set() for name, _ in self._slices
+            }
+
+            for record in records:
+                with tracing.span("evaluate_record"):
+                    logger.debug(
+                        "Evaluating sample %s with %s metric(s)",
+                        record.task.metadata.get("dataset_id")
+                        or record.task.metadata.get("sample_id"),
+                        len(self._metrics),
+                    )
+                    strategy = self._strategy_resolver(record)
+                    task_metadata = record.task.metadata
+                    sample_id = task_metadata.get("dataset_id") or task_metadata.get(
+                        "sample_id"
+                    )
+                    for name, fn in self._slices:
+                        try:
+                            if fn(record) and sample_id is not None:
+                                slice_members[name].add(sample_id)
+                        except Exception:
+                            pass
+                    eval_items = list(strategy.prepare(record))
+                    item_scores: list[core_entities.MetricScore] = []
+                    record_failures: list[str] = []
+
+                    for item in eval_items:
+                        if item.record.output is None:
+                            message = "Missing model output"
+                            failures.append(
+                                EvaluationFailure(sample_id=sample_id, message=message)
+                            )
+                            record_failures.append(message)
+                            continue
+                        try:
+                            with tracing.span("extract"):
+                                prediction = self._extractor.extract(
+                                    item.record.output.text
+                                )
+                        except extractors.FieldExtractionError as exc:
+                            message = str(exc)
+                            failures.append(
+                                EvaluationFailure(sample_id=sample_id, message=message)
+                            )
+                            record_failures.append(message)
+                            continue
+
+                        # CRITICAL: Always call reference_selector if provided (takes precedence)
+                        # This fixes the issue where DefaultEvaluationStrategy's reference
+                        # would prevent custom reference_selector from being called
+                        if self._has_custom_reference_selector:
+                            reference = self._reference_selector(record)
+                        elif item.reference is not None:
+                            reference = item.reference
+                        else:
+                            reference = _default_reference_selector(record)
+
+                        references = (
+                            _normalize_references(reference)
+                            if reference is not None
+                            else []
+                        )
+                        metadata = {"sample_id": sample_id}
+                        extract_start = time.perf_counter()
+                        item_scores_for_item: list[core_entities.MetricScore] = []
+                        for metric in self._metrics:
+                            requires_reference = getattr(
+                                metric, "requires_reference", True
+                            )
+                            if requires_reference and not references:
+                                message = (
+                                    f"Missing reference for metric '{metric.name}'"
+                                )
+                                failures.append(
+                                    EvaluationFailure(
+                                        sample_id=sample_id, message=message
+                                    )
+                                )
+                                record_failures.append(message)
+                                continue
+                            metric_start = time.perf_counter()
+                            try:
+                                with tracing.span(
+                                    "compute_metric", metric_name=metric.name
+                                ):
+                                    score = metric.compute(
+                                        prediction=prediction,
+                                        references=references,
+                                        metadata=metadata,
+                                    )
+                                score.metadata["evaluation_time_ms"] = (
+                                    time.perf_counter() - metric_start
+                                ) * 1000
+                                item_scores_for_item.append(score)
+                            except Exception as exc:  # pragma: no cover - guarded
+                                message = (
+                                    f"Metric '{metric.name}' failed for sample {sample_id}: {exc}"
+                                )
+                                logger.warning(message)
+                                failures.append(
+                                    EvaluationFailure(
+                                        sample_id=sample_id, message=message
+                                    )
+                                )
+                                record_failures.append(message)
+                        extraction_duration = (
+                            time.perf_counter() - extract_start
+                        ) * 1000
+                        for score in item_scores_for_item:
+                            score.metadata.setdefault(
+                                "extraction_time_ms", extraction_duration
+                            )
+                        item_scores.extend(item_scores_for_item)
+
+                    aggregated_scores = strategy.aggregate(record, item_scores)
+                    for score in aggregated_scores:
+                        per_metric[score.metric_name].append(score)
+                    per_record.append(
+                        core_entities.EvaluationRecord(
+                            sample_id=sample_id,
+                            scores=aggregated_scores,
+                            failures=record_failures,
+                        )
+                    )
+
+            aggregates = {
+                name: MetricAggregate.from_scores(name, scores)
+                for name, scores in per_metric.items()
+            }
+
+            return EvaluationReport(
+                metrics=aggregates,
+                failures=failures,
+                records=per_record,
+                slices=self._compute_slice_aggregates(per_metric, slice_members),
+            )
+
+    def register_slice(
+        self, name: str, fn: Callable[[core_entities.GenerationRecord], bool]
+    ) -> None:
+        """Register a slice for subset analysis.
+
+        Args:
+            name: Slice name
+            fn: Predicate function to determine slice membership
+        """
+        self._slices.append((name, fn))
+
+    def _compute_slice_aggregates(
+        self,
+        per_metric: dict[str, list[core_entities.MetricScore]],
+        slice_members: dict[str, set[str]],
+    ) -> dict[str, dict[str, MetricAggregate]]:
+        """Compute metric aggregates for each slice.
+
+        Args:
+            per_metric: Scores by metric name
+            slice_members: Sample IDs by slice name
+
+        Returns:
+            Nested dict of slice -> metric -> aggregate
+        """
+        if not slice_members:
+            return {}
+        slice_aggregates: dict[str, dict[str, MetricAggregate]] = {}
+        for name, members in slice_members.items():
+            slice_scores_by_metric: dict[str, list[core_entities.MetricScore]] = {}
+            for metric_name, scores in per_metric.items():
+                filtered = [s for s in scores if s.metadata.get("sample_id") in members]
+                slice_scores_by_metric[metric_name] = filtered
+            slice_aggregates[name] = {
+                metric_name: MetricAggregate.from_scores(metric_name, scores)
+                for metric_name, scores in slice_scores_by_metric.items()
+            }
+        return slice_aggregates

themis/evaluation/reports.py

@@ -0,0 +1,293 @@
+"""Evaluation report data structures."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from statistics import mean
+from typing import Dict, List, Literal, Sequence
+
+from themis.core import entities as core_entities
+from themis.evaluation.statistics import (
+    bootstrap_ci,
+    cohens_d,
+    cohens_h,
+    holm_bonferroni,
+    paired_permutation_test,
+    paired_t_test,
+    permutation_test,
+)
+from themis.evaluation.statistics.types import (
+    BootstrapResult,
+    ComparisonResult,
+    EffectSize,
+    PermutationTestResult,
+)
+
+
+@dataclass
+class EvaluationFailure:
+    sample_id: str | None
+    message: str
+
+
+@dataclass
+class MetricAggregate:
+    name: str
+    count: int
+    mean: float
+    per_sample: List[core_entities.MetricScore]
+
+    @classmethod
+    def from_scores(
+        cls, name: str, scores: List[core_entities.MetricScore]
+    ) -> "MetricAggregate":
+        if not scores:
+            return cls(name=name, count=0, mean=0.0, per_sample=[])
+        return cls(
+            name=name,
+            count=len(scores),
+            mean=mean(score.value for score in scores),
+            per_sample=scores,
+        )
+
+
+@dataclass
+class EvaluationReport:
+    metrics: dict[str, MetricAggregate]
+    failures: List[EvaluationFailure]
+    records: List[core_entities.EvaluationRecord]
+    slices: dict[str, dict[str, MetricAggregate]] = field(default_factory=dict)
+
+
+def _metric_values(report: EvaluationReport, metric_name: str) -> list[float]:
+    agg = report.metrics.get(metric_name)
+    if not agg:
+        return []
+    return [s.value for s in agg.per_sample]
+
+
+def _metric_values_by_sample(
+    report: EvaluationReport, metric_name: str
+) -> dict[str, float]:
+    values: dict[str, float] = {}
+    for record in report.records:
+        if not record.sample_id:
+            continue
+        for score in record.scores:
+            if score.metric_name == metric_name:
+                values[record.sample_id] = score.value
+                break
+    return values
+
+
+def aligned_metric_values(
+    report_a: EvaluationReport, report_b: EvaluationReport, metric_name: str
+) -> tuple[list[float], list[float]]:
+    values_a = _metric_values_by_sample(report_a, metric_name)
+    values_b = _metric_values_by_sample(report_b, metric_name)
+    common_ids = sorted(set(values_a) & set(values_b))
+    if not common_ids:
+        raise ValueError(f"No overlapping sample_ids for metric '{metric_name}'")
+    aligned_a = [values_a[sample_id] for sample_id in common_ids]
+    aligned_b = [values_b[sample_id] for sample_id in common_ids]
+    return aligned_a, aligned_b
+
+
+def ci_for_metric(
+    report: EvaluationReport,
+    metric_name: str,
+    confidence_level: float = 0.95,
+    n_bootstrap: int = 10000,
+) -> BootstrapResult:
+    values = _metric_values(report, metric_name)
+    if not values:
+        raise ValueError(f"No scores for metric '{metric_name}'")
+    return bootstrap_ci(
+        values, n_bootstrap=n_bootstrap, confidence_level=confidence_level
+    )
+
+
+def permutation_test_for_metric(
+    report_a: EvaluationReport,
+    report_b: EvaluationReport,
+    metric_name: str,
+    statistic: Literal["mean_diff", "median_diff"] = "mean_diff",
+    n_permutations: int = 10000,
+    seed: int | None = None,
+    align_by_sample_id: bool = True,
+) -> PermutationTestResult:
+    if align_by_sample_id:
+        values_a, values_b = aligned_metric_values(report_a, report_b, metric_name)
+    else:
+        values_a = _metric_values(report_a, metric_name)
+        values_b = _metric_values(report_b, metric_name)
+    if not values_a or not values_b:
+        raise ValueError(f"Both reports must have scores for metric '{metric_name}'")
+    return permutation_test(
+        values_a,
+        values_b,
+        statistic=statistic,
+        n_permutations=n_permutations,
+        seed=seed,
+    )
+
+
+def paired_permutation_test_for_metric(
+    report_a: EvaluationReport,
+    report_b: EvaluationReport,
+    metric_name: str,
+    statistic: Literal["mean_diff", "median_diff"] = "mean_diff",
+    n_permutations: int = 10000,
+    seed: int | None = None,
+) -> PermutationTestResult:
+    values_a, values_b = aligned_metric_values(report_a, report_b, metric_name)
+    return paired_permutation_test(
+        values_a,
+        values_b,
+        statistic=statistic,
+        n_permutations=n_permutations,
+        seed=seed,
+    )
+
+
+def cohens_h_for_metric(
+    report_a: EvaluationReport,
+    report_b: EvaluationReport,
+    metric_name: str,
+) -> EffectSize:
+    agg_a = report_a.metrics.get(metric_name)
+    agg_b = report_b.metrics.get(metric_name)
+    if not agg_a or not agg_b:
+        raise ValueError(f"Both reports must have aggregate for metric '{metric_name}'")
+    return cohens_h(agg_a.mean, agg_b.mean)
+
+
+def cohens_d_for_metric(
+    report_a: EvaluationReport,
+    report_b: EvaluationReport,
+    metric_name: str,
+) -> EffectSize:
+    values_a, values_b = aligned_metric_values(report_a, report_b, metric_name)
+    if len(values_a) < 2 or len(values_b) < 2:
+        raise ValueError("Each group must have at least 2 values for Cohen's d")
+    return cohens_d(values_a, values_b)
+
+
+def paired_t_test_for_metric(
+    report_a: EvaluationReport,
+    report_b: EvaluationReport,
+    metric_name: str,
+    significance_level: float = 0.05,
+) -> ComparisonResult:
+    values_a, values_b = aligned_metric_values(report_a, report_b, metric_name)
+    result = paired_t_test(values_a, values_b, significance_level=significance_level)
+    return ComparisonResult(
+        metric_name=metric_name,
+        baseline_mean=result.baseline_mean,
+        treatment_mean=result.treatment_mean,
+        difference=result.difference,
+        relative_change=result.relative_change,
+        t_statistic=result.t_statistic,
+        p_value=result.p_value,
+        is_significant=result.is_significant,
+        baseline_ci=result.baseline_ci,
+        treatment_ci=result.treatment_ci,
+    )
+
+
+def _slice_metric_values(
+    report: EvaluationReport, slice_name: str, metric_name: str
+) -> list[float]:
+    slice_map = report.slices.get(slice_name)
+    if not slice_map:
+        return []
+    agg = slice_map.get(metric_name)
+    if not agg:
+        return []
+    return [s.value for s in agg.per_sample]
+
+
+def ci_for_slice_metric(
+    report: EvaluationReport,
+    slice_name: str,
+    metric_name: str,
+    confidence_level: float = 0.95,
+    n_bootstrap: int = 10000,
+) -> BootstrapResult:
+    values = _slice_metric_values(report, slice_name, metric_name)
+    if not values:
+        raise ValueError(
+            f"No scores for metric '{metric_name}' in slice '{slice_name}'"
+        )
+    return bootstrap_ci(
+        values, n_bootstrap=n_bootstrap, confidence_level=confidence_level
+    )
+
+
+def compare_reports_with_holm(
+    report_a: EvaluationReport,
+    report_b: EvaluationReport,
+    metric_names: Sequence[str],
+    statistic: Literal["mean_diff", "median_diff"] = "mean_diff",
+    n_permutations: int = 10000,
+    seed: int | None = None,
+    paired: bool = True,
+) -> Dict[str, object]:
+    p_values: list[float] = []
+    pt_results: Dict[str, PermutationTestResult] = {}
+    for name in metric_names:
+        if paired:
+            pt = paired_permutation_test_for_metric(
+                report_a,
+                report_b,
+                name,
+                statistic=statistic,
+                n_permutations=n_permutations,
+                seed=seed,
+            )
+        else:
+            pt = permutation_test_for_metric(
+                report_a,
+                report_b,
+                name,
+                statistic=statistic,
+                n_permutations=n_permutations,
+                seed=seed,
+                align_by_sample_id=True,
+            )
+        pt_results[name] = pt
+        p_values.append(pt.p_value)
+    corrected = holm_bonferroni(p_values)
+    return {
+        "per_metric": pt_results,
+        "holm_significant": corrected,
+    }
+
+
+def confusion_matrix(
+    labels_true: Sequence[str], labels_pred: Sequence[str]
+) -> Dict[str, Dict[str, int]]:
+    if len(labels_true) != len(labels_pred):
+        raise ValueError("labels_true and labels_pred must have same length")
+    cm: Dict[str, Dict[str, int]] = {}
+    for t, p in zip(labels_true, labels_pred):
+        cm.setdefault(t, {})
+        cm[t][p] = cm[t].get(p, 0) + 1
+    return cm
+
+
+__all__ = [
+    "EvaluationFailure",
+    "MetricAggregate",
+    "EvaluationReport",
+    "aligned_metric_values",
+    "ci_for_metric",
+    "ci_for_slice_metric",
+    "permutation_test_for_metric",
+    "paired_permutation_test_for_metric",
+    "cohens_h_for_metric",
+    "cohens_d_for_metric",
+    "paired_t_test_for_metric",
+    "confusion_matrix",
+    "compare_reports_with_holm",
+]

themis/evaluation/statistics/__init__.py

@@ -0,0 +1,53 @@
+"""Statistical analysis utilities for experiment evaluation results.
+
+This module provides statistical analysis tools for computing confidence intervals,
+significance tests, and statistical comparisons across experiment runs.
+"""
+
+from __future__ import annotations
+
+from .bootstrap import bootstrap_ci
+from .confidence_intervals import (
+    compute_confidence_interval,
+    compute_statistical_summary,
+)
+from .effect_sizes import cohens_d, cohens_h
+from .hypothesis_tests import (
+    compare_metrics,
+    holm_bonferroni,
+    paired_permutation_test,
+    paired_t_test,
+    permutation_test,
+)
+from .types import (
+    BootstrapResult,
+    ComparisonResult,
+    ConfidenceInterval,
+    EffectSize,
+    PermutationTestResult,
+    StatisticalSummary,
+)
+
+__all__ = [
+    # Types
+    "ConfidenceInterval",
+    "StatisticalSummary",
+    "ComparisonResult",
+    "PermutationTestResult",
+    "BootstrapResult",
+    "EffectSize",
+    # Confidence intervals
+    "compute_confidence_interval",
+    "compute_statistical_summary",
+    # Hypothesis tests
+    "compare_metrics",
+    "permutation_test",
+    "paired_permutation_test",
+    "paired_t_test",
+    "holm_bonferroni",
+    # Bootstrap
+    "bootstrap_ci",
+    # Effect sizes
+    "cohens_h",
+    "cohens_d",
+]