themis-eval 0.1.0__py3-none-any.whl → 0.2.0__py3-none-any.whl

themis/evaluation/conditional.py

@@ -0,0 +1,410 @@
+"""Conditional and adaptive evaluation strategies.
+
+This module provides evaluation components that adapt based on sample characteristics:
+- ConditionalMetric: Only runs when condition is met
+- AdaptiveEvaluationPipeline: Selects metrics based on sample metadata
+- Metric selectors: Helper functions for common selection patterns
+
+Example:
+    >>> # Only run math verification on math problems
+    >>> math_metric = ConditionalMetric(
+    ...     metric=MathVerifyAccuracy(),
+    ...     condition=lambda record: record.task.metadata.get("type") == "math"
+    ... )
+    >>>
+    >>> # Adaptively select metrics based on task type
+    >>> def select_metrics(record):
+    ...     if record.task.metadata.get("type") == "math":
+    ...         return [ExactMatch(), MathVerifyAccuracy()]
+    ...     return [ExactMatch()]
+    >>>
+    >>> pipeline = AdaptiveEvaluationPipeline(
+    ...     extractor=extractor,
+    ...     metric_selector=select_metrics
+    ... )
+"""
+
+from __future__ import annotations
+
+from collections import defaultdict
+from dataclasses import dataclass
+from typing import Any, Callable, Sequence
+
+from themis.core import entities as core_entities
+from themis.evaluation import pipeline, reports
+from themis.interfaces import Metric
+from themis.utils import tracing
+
+
+@dataclass
+class ConditionalMetric:
+    """Metric that only runs when condition is met.
+
+    This wrapper allows you to conditionally apply metrics based on
+    record characteristics (metadata, task type, etc.).
+
+    Attributes:
+        metric: Wrapped metric
+        condition: Function that determines if metric should run
+        default_score: Score to return when condition is False
+        name: Optional override for metric name
+
+    Example:
+        >>> # Only run expensive metric on hard problems
+        >>> hard_metric = ConditionalMetric(
+        ...     metric=ExpensiveVerification(),
+        ...     condition=lambda r: r.task.metadata.get("difficulty") == "hard",
+        ...     default_score=0.0
+        ... )
+    """
+
+    metric: Metric
+    condition: Callable[[core_entities.GenerationRecord], bool]
+    default_score: float = 0.0
+    name: str | None = None
+
+    def __post_init__(self):
+        if self.name is None:
+            self.name = f"conditional_{self.metric.name}"
+
+    def should_evaluate(self, record: core_entities.GenerationRecord) -> bool:
+        """Check if metric should be evaluated for this record.
+
+        Args:
+            record: Generation record
+
+        Returns:
+            True if condition is met
+        """
+        try:
+            return self.condition(record)
+        except Exception:
+            # If condition check fails, don't run metric
+            return False
+
+    def compute(
+        self,
+        *,
+        prediction: Any,
+        references: Sequence[Any],
+        metadata: dict[str, Any] | None = None,
+    ) -> core_entities.MetricScore:
+        """Compute metric score.
+
+        Note: This method doesn't check the condition - it's assumed
+        the condition was already checked before calling compute.
+
+        Args:
+            prediction: Predicted value
+            references: Reference values
+            metadata: Optional metadata
+
+        Returns:
+            Metric score
+        """
+        return self.metric.compute(
+            prediction=prediction,
+            references=references,
+            metadata=metadata,
+        )
+
+    def compute_or_default(
+        self,
+        record: core_entities.GenerationRecord,
+        *,
+        prediction: Any,
+        references: Sequence[Any],
+        metadata: dict[str, Any] | None = None,
+    ) -> core_entities.MetricScore:
+        """Compute metric or return default score if condition not met.
+
+        Args:
+            record: Generation record (for condition check)
+            prediction: Predicted value
+            references: Reference values
+            metadata: Optional metadata
+
+        Returns:
+            Metric score or default
+        """
+        if self.should_evaluate(record):
+            return self.compute(
+                prediction=prediction,
+                references=references,
+                metadata=metadata,
+            )
+        else:
+            return core_entities.MetricScore(
+                metric_name=self.name or self.metric.name,
+                value=self.default_score,
+                metadata={"skipped": True, "reason": "condition_not_met"},
+            )
+
+
+class AdaptiveEvaluationPipeline(pipeline.EvaluationPipeline):
+    """Pipeline that selects metrics based on sample characteristics.
+
+    This pipeline allows different metrics to be applied to different
+    samples based on their metadata, task type, or other characteristics.
+
+    This is more efficient than ConditionalMetric when you have many
+    samples that can be grouped by their metric requirements.
+
+    Example:
+        >>> def select_metrics(record):
+        ...     task_type = record.task.metadata.get("type")
+        ...     if task_type == "math":
+        ...         return [ExactMatch(), MathVerifyAccuracy()]
+        ...     elif task_type == "code":
+        ...         return [CodeExecutionMetric()]
+        ...     return [ExactMatch()]
+        >>>
+        >>> pipeline = AdaptiveEvaluationPipeline(
+        ...     extractor=extractor,
+        ...     metric_selector=select_metrics
+        ... )
+    """
+
+    def __init__(
+        self,
+        *,
+        extractor: Any,
+        metric_selector: Callable[[core_entities.GenerationRecord], list[Metric]],
+        **kwargs: Any,
+    ):
+        """Initialize adaptive pipeline.
+
+        Args:
+            extractor: Extractor for all samples
+            metric_selector: Function that selects metrics for each record
+            **kwargs: Additional arguments passed to EvaluationPipeline
+        """
+        # Initialize with empty metrics - we'll select them dynamically
+        super().__init__(extractor=extractor, metrics=[], **kwargs)
+        self._metric_selector = metric_selector
+
+    def evaluate(
+        self, records: Sequence[core_entities.GenerationRecord]
+    ) -> pipeline.EvaluationReport:
+        """Evaluate records with adaptive metric selection.
+
+        Args:
+            records: Generation records to evaluate
+
+        Returns:
+            Evaluation report
+        """
+        with tracing.span("adaptive_evaluation", num_records=len(records)):
+            # Group records by which metrics apply
+            metric_groups: dict[
+                tuple[str, ...], list[core_entities.GenerationRecord]
+            ] = defaultdict(list)
+            record_metrics: dict[str, list[Metric]] = {}
+
+            # Phase 1: Group records by metric set
+            with tracing.span("group_by_metrics"):
+                for record in records:
+                    selected_metrics = self._metric_selector(record)
+                    metric_key = tuple(m.name for m in selected_metrics)
+                    metric_groups[metric_key].append(record)
+
+                    # Store mapping for later
+                    sample_id = str(record.task.metadata.get("dataset_id", "unknown"))
+                    record_metrics[sample_id] = selected_metrics
+
+            # Phase 2: Evaluate each group with appropriate metrics
+            all_eval_records = []
+            with tracing.span("evaluate_groups", num_groups=len(metric_groups)):
+                for metric_key, group_records in metric_groups.items():
+                    if not group_records:
+                        continue
+
+                    # Get metrics for this group
+                    sample_id = str(
+                        group_records[0].task.metadata.get("dataset_id", "unknown")
+                    )
+                    group_metrics = record_metrics.get(sample_id, [])
+
+                    with tracing.span(
+                        "evaluate_group",
+                        metric_names=list(metric_key),
+                        num_records=len(group_records),
+                    ):
+                        # Create temporary pipeline for this group
+                        temp_pipeline = pipeline.EvaluationPipeline(
+                            extractor=self._extractor,
+                            metrics=group_metrics,
+                        )
+
+                        # Evaluate group
+                        group_report = temp_pipeline.evaluate(group_records)
+                        all_eval_records.extend(group_report.records)
+
+            # Phase 3: Aggregate all results
+            with tracing.span("aggregate_adaptive_results"):
+                # Collect all metric scores by metric name
+                metric_scores_by_name: dict[str, list[core_entities.MetricScore]] = (
+                    defaultdict(list)
+                )
+                for eval_record in all_eval_records:
+                    for score_record in eval_record.scores:
+                        metric_scores_by_name[score_record.metric_name].append(
+                            score_record
+                        )
+
+                # Compute aggregates
+                metric_aggregates = {}
+                for metric_name, score_objs in metric_scores_by_name.items():
+                    if score_objs:
+                        metric_aggregates[metric_name] = (
+                            reports.MetricAggregate.from_scores(
+                                name=metric_name,
+                                scores=score_objs,
+                            )
+                        )
+
+                return reports.EvaluationReport(
+                    metrics=metric_aggregates,
+                    failures=[],  # No failures tracked in adaptive pipeline
+                    records=all_eval_records,
+                )
+
+
+# Helper functions for common metric selection patterns
+
+
+def select_by_metadata_field(
+    field: str, metric_map: dict[Any, list[Metric]], default: list[Metric] | None = None
+) -> Callable[[core_entities.GenerationRecord], list[Metric]]:
+    """Create selector that chooses metrics based on metadata field value.
+
+    Args:
+        field: Metadata field to check
+        metric_map: Mapping from field value to metrics
+        default: Default metrics if field value not in map
+
+    Returns:
+        Metric selector function
+
+    Example:
+        >>> selector = select_by_metadata_field(
+        ...     "type",
+        ...     {
+        ...         "math": [ExactMatch(), MathVerifyAccuracy()],
+        ...         "code": [CodeExecutionMetric()],
+        ...     },
+        ...     default=[ExactMatch()]
+        ... )
+    """
+    default_metrics = default or []
+
+    def selector(record: core_entities.GenerationRecord) -> list[Metric]:
+        value = record.task.metadata.get(field)
+        return metric_map.get(value, default_metrics)
+
+    return selector
+
+
+def select_by_difficulty(
+    easy_metrics: list[Metric],
+    medium_metrics: list[Metric],
+    hard_metrics: list[Metric],
+    difficulty_field: str = "difficulty",
+) -> Callable[[core_entities.GenerationRecord], list[Metric]]:
+    """Create selector that chooses metrics based on difficulty.
+
+    Args:
+        easy_metrics: Metrics for easy problems
+        medium_metrics: Metrics for medium problems
+        hard_metrics: Metrics for hard problems
+        difficulty_field: Name of difficulty field in metadata
+
+    Returns:
+        Metric selector function
+
+    Example:
+        >>> selector = select_by_difficulty(
+        ...     easy_metrics=[ExactMatch()],
+        ...     medium_metrics=[ExactMatch(), PartialCredit()],
+        ...     hard_metrics=[ExactMatch(), PartialCredit(), ManualReview()]
+        ... )
+    """
+    return select_by_metadata_field(
+        difficulty_field,
+        {
+            "easy": easy_metrics,
+            "medium": medium_metrics,
+            "hard": hard_metrics,
+        },
+        default=medium_metrics,
+    )
+
+
+def select_by_condition(
+    condition: Callable[[core_entities.GenerationRecord], bool],
+    metrics_if_true: list[Metric],
+    metrics_if_false: list[Metric],
+) -> Callable[[core_entities.GenerationRecord], list[Metric]]:
+    """Create selector based on arbitrary condition.
+
+    Args:
+        condition: Function to determine which metrics to use
+        metrics_if_true: Metrics if condition is True
+        metrics_if_false: Metrics if condition is False
+
+    Returns:
+        Metric selector function
+
+    Example:
+        >>> selector = select_by_condition(
+        ...     lambda r: len(r.output.text) > 1000,
+        ...     metrics_if_true=[SummaryMetrics()],
+        ...     metrics_if_false=[ExactMatch()]
+        ... )
+    """
+
+    def selector(record: core_entities.GenerationRecord) -> list[Metric]:
+        try:
+            if condition(record):
+                return metrics_if_true
+            else:
+                return metrics_if_false
+        except Exception:
+            # If condition fails, use false branch
+            return metrics_if_false
+
+    return selector
+
+
+def combine_selectors(
+    *selectors: Callable[[core_entities.GenerationRecord], list[Metric]],
+) -> Callable[[core_entities.GenerationRecord], list[Metric]]:
+    """Combine multiple selectors (union of their metrics).
+
+    Args:
+        *selectors: Metric selectors to combine
+
+    Returns:
+        Combined selector that returns union of all selected metrics
+
+    Example:
+        >>> selector = combine_selectors(
+        ...     select_by_type,
+        ...     select_by_difficulty,
+        ... )
+    """
+
+    def combined(record: core_entities.GenerationRecord) -> list[Metric]:
+        all_metrics = []
+        seen_names = set()
+
+        for selector in selectors:
+            selected = selector(record)
+            for metric in selected:
+                if metric.name not in seen_names:
+                    all_metrics.append(metric)
+                    seen_names.add(metric.name)
+
+        return all_metrics
+
+    return combined

themis/evaluation/extractors/__init__.py

@@ -0,0 +1,19 @@
+"""Output extractors used during evaluation."""
+
+from __future__ import annotations
+
+from .error_taxonomy_extractor import ErrorTaxonomyExtractor
+from .exceptions import FieldExtractionError
+from .identity_extractor import IdentityExtractor
+from .json_field_extractor import JsonFieldExtractor
+from .math_verify_extractor import MathVerifyExtractor
+from .regex_extractor import RegexExtractor
+
+__all__ = [
+    "FieldExtractionError",
+    "JsonFieldExtractor",
+    "RegexExtractor",
+    "IdentityExtractor",
+    "MathVerifyExtractor",
+    "ErrorTaxonomyExtractor",
+]

themis/evaluation/extractors/error_taxonomy_extractor.py

@@ -0,0 +1,80 @@
+from __future__ import annotations
+
+import re
+from typing import Dict
+
+
+class ErrorTaxonomyExtractor:
+    """
+    Lightweight error taxonomy extractor.
+
+    Heuristics:
+    - format_parse_failure: Unbalanced JSON-like braces suggest parsing intent but malformed format
+    - arithmetic_slip: Simple arithmetic expression like "X + Y = Z" evaluated incorrectly
+    - reasoning_gap: Final answer given without common justification keywords
+    """
+
+    def extract(self, text: str) -> Dict[str, bool]:
+        labels = {
+            "format_parse_failure": False,
+            "arithmetic_slip": False,
+            "reasoning_gap": False,
+        }
+
+        # format_parse_failure: JSON-like but malformed braces
+        if ("{" in text or "}" in text) and not self._balanced_braces(text):
+            labels["format_parse_failure"] = True
+
+        # arithmetic_slip: pattern "A op B = Z" mismatch
+        try:
+            if self._has_arithmetic_mismatch(text):
+                labels["arithmetic_slip"] = True
+        except Exception:
+            pass
+
+        # reasoning_gap: answer provided without justification keywords
+        lowered = text.lower()
+        has_answer_phrase = any(p in lowered for p in ("answer", "final", "therefore"))
+        has_justification = any(
+            k in lowered for k in ("because", "since", "thus", "therefore", "reason")
+        )
+        if has_answer_phrase and not has_justification:
+            labels["reasoning_gap"] = True
+
+        return labels
+
+    def _balanced_braces(self, text: str) -> bool:
+        count = 0
+        for ch in text:
+            if ch == "{":
+                count += 1
+            elif ch == "}":
+                count -= 1
+                if count < 0:
+                    return False
+        return count == 0
+
+    def _has_arithmetic_mismatch(self, text: str) -> bool:
+        pattern = (
+            r"(-?\d+(?:\.\d+)?)\s*([+\-*/])\s*(-?\d+(?:\.\d+)?)\s*=\s*(-?\d+(?:\.\d+)?)"
+        )
+        m = re.search(pattern, text)
+        if not m:
+            return False
+        a, op, b, z = m.groups()
+        a = float(a)
+        b = float(b)
+        z = float(z)
+        if op == "+":
+            calc = a + b
+        elif op == "-":
+            calc = a - b
+        elif op == "*":
+            calc = a * b
+        elif op == "/":
+            if b == 0:
+                return False
+            calc = a / b
+        else:
+            return False
+        return abs(calc - z) > 1e-9

themis/evaluation/extractors/exceptions.py

@@ -0,0 +1,7 @@
+"""Extractor exceptions."""
+
+from __future__ import annotations
+
+
+class FieldExtractionError(RuntimeError):
+    """Raised when an output field cannot be extracted."""

themis/evaluation/extractors/identity_extractor.py

@@ -0,0 +1,29 @@
+"""Identity (pass-through) extraction."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+@dataclass
+class IdentityExtractor:
+    """Extractor that returns the raw output as-is.
+
+    Args:
+        strip_whitespace: Whether to strip leading/trailing whitespace
+    """
+
+    strip_whitespace: bool = True
+
+    def extract(self, raw_output: str) -> str:
+        """Return the raw output, optionally stripping whitespace.
+
+        Args:
+            raw_output: Raw output from model
+
+        Returns:
+            Raw output (possibly stripped)
+        """
+        if self.strip_whitespace:
+            return raw_output.strip()
+        return raw_output

themis/evaluation/extractors/json_field_extractor.py

@@ -0,0 +1,45 @@
+"""JSON field extraction."""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from typing import Any
+
+from .exceptions import FieldExtractionError
+
+
+@dataclass
+class JsonFieldExtractor:
+    """Extracts a specific field from JSON output.
+
+    Args:
+        field_path: Dot-separated path to the field (e.g., "answer" or "result.value")
+    """
+
+    field_path: str
+
+    def extract(self, raw_output: str) -> Any:
+        """Extract the specified field from JSON output.
+
+        Args:
+            raw_output: Raw JSON string from model
+
+        Returns:
+            Extracted field value
+
+        Raises:
+            FieldExtractionError: If JSON is invalid or field is missing
+        """
+        try:
+            payload = json.loads(raw_output)
+        except json.JSONDecodeError as exc:  # pragma: no cover - defensive path
+            raise FieldExtractionError("Invalid JSON output") from exc
+
+        current = payload
+        for part in self.field_path.split("."):
+            if isinstance(current, dict) and part in current:
+                current = current[part]
+            else:
+                raise FieldExtractionError(f"Missing field '{self.field_path}'")
+        return current

themis/evaluation/extractors/math_verify_extractor.py

@@ -0,0 +1,37 @@
+"""Math-verify extraction for mathematical expressions."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from .exceptions import FieldExtractionError
+
+
+@dataclass
+class MathVerifyExtractor:
+    """Extracts the final boxed answer using math-verify parsing.
+
+    This extractor uses the math-verify library to parse and normalize
+    mathematical expressions from LaTeX boxed notation.
+    """
+
+    def extract(self, raw_output: str) -> str:
+        """Extract and parse boxed mathematical answer.
+
+        Args:
+            raw_output: Raw output containing \\boxed{...} notation
+
+        Returns:
+            Parsed and normalized mathematical expression
+
+        Raises:
+            FieldExtractionError: If math-verify parsing fails
+        """
+        from themis.evaluation import math_verify_utils as mv_utils
+
+        candidate = mv_utils.extract_last_boxed(raw_output)
+        try:
+            parsed = mv_utils.parse_expression(candidate)
+        except Exception as exc:  # pragma: no cover - parse failure
+            raise FieldExtractionError("math-verify parsing failed") from exc
+        return str(parsed).strip()

themis/evaluation/extractors/regex_extractor.py

@@ -0,0 +1,43 @@
+"""Regex-based extraction."""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from typing import Dict
+
+from .exceptions import FieldExtractionError
+
+
+@dataclass
+class RegexExtractor:
+    """Extracts fields using regular expression patterns.
+
+    Args:
+        pattern: Regular expression pattern with optional named groups
+    """
+
+    pattern: str
+
+    def __post_init__(self) -> None:
+        self._compiled = re.compile(self.pattern)
+
+    def extract(self, text: str) -> Dict[str, str]:
+        """Extract fields from text using regex pattern.
+
+        Args:
+            text: Text to extract from
+
+        Returns:
+            Dictionary of extracted groups (named or numbered)
+
+        Raises:
+            FieldExtractionError: If pattern does not match
+        """
+        match = self._compiled.search(text)
+        if not match:
+            raise FieldExtractionError("Regex did not match")
+        groups = match.groupdict()
+        if groups:
+            return {key: value.strip() for key, value in groups.items()}
+        return {str(index): value.strip() for index, value in enumerate(match.groups())}