themis-eval 0.1.1__py3-none-any.whl → 0.2.0__py3-none-any.whl

themis/integrations/huggingface.py

@@ -3,8 +3,15 @@ from __future__ import annotations
 import json
 from dataclasses import asdict, is_dataclass
 from pathlib import Path
+from typing import TYPE_CHECKING
 
-from huggingface_hub import HfApi
+if TYPE_CHECKING:
+    from huggingface_hub import HfApi
+else:
+    try:
+        from huggingface_hub import HfApi
+    except ImportError:
+        HfApi = None  # type: ignore
 
 from themis.config.schema import HuggingFaceHubConfig
 from themis.core.entities import ExperimentReport
@@ -24,6 +31,10 @@ def to_dict(obj):
 
 class HuggingFaceHubUploader:
     def __init__(self, config: HuggingFaceHubConfig):
+        if HfApi is None:
+            raise ImportError(
+                "huggingface_hub is not installed. Install with: pip install huggingface_hub"
+            )
         self.config = config
         self.api = HfApi()
 

themis/integrations/wandb.py

@@ -1,6 +1,14 @@
 from __future__ import annotations
 
-import wandb
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    import wandb
+else:
+    try:
+        import wandb
+    except ImportError:
+        wandb = None  # type: ignore
 
 from themis.config.schema import WandbConfig
 from themis.core.entities import ExperimentReport
@@ -8,6 +16,10 @@ from themis.core.entities import ExperimentReport
 
 class WandbTracker:
     def __init__(self, config: WandbConfig):
+        if wandb is None:
+            raise ImportError(
+                "wandb is not installed. Install with: pip install wandb"
+            )
         self.config = config
 
     def init(self, experiment_config: dict) -> None:

themis/interfaces/__init__.py

@@ -56,11 +56,67 @@ class DatasetAdapter(Protocol):
 
 
 class Extractor(Protocol):
+    """Protocol for extractors that parse model output.
+
+    Extractors are responsible for parsing raw model output text and
+    extracting the relevant answer or prediction. The evaluation pipeline
+    calls the extractor before passing the result to metrics.
+
+    Example:
+        >>> class JsonExtractor:
+        ...     def extract(self, raw_output: str) -> Any:
+        ...         import json
+        ...         return json.loads(raw_output)["answer"]
+    """
+
     def extract(self, raw_output: str) -> Any:  # pragma: no cover - protocol
+        """Extract prediction from raw model output.
+
+        Args:
+            raw_output: Raw text output from the model
+
+        Returns:
+            Extracted prediction (type depends on extractor implementation)
+
+        Raises:
+            FieldExtractionError: If extraction fails
+        """
         ...
 
 
 class Metric(ABC):
+    """Abstract base class for evaluation metrics.
+
+    Metrics compute scores by comparing model predictions against reference values.
+    The evaluation pipeline handles extraction before passing data to metrics.
+
+    IMPORTANT - Extractor Contract:
+        The 'prediction' parameter receives EXTRACTED output from the extractor,
+        NOT raw model output. Metrics should NOT attempt to re-extract or parse
+        the prediction - it has already been processed by the pipeline's extractor.
+
+        Example flow:
+            1. Model generates: "<think>reasoning</think><answer>42</answer>"
+            2. Extractor extracts: "42"
+            3. Metric receives: prediction="42" (already extracted)
+
+    Attributes:
+        name: Unique metric identifier
+        requires_reference: Whether metric needs reference values (default: True)
+
+    Example:
+        >>> class ExactMatch(Metric):
+        ...     name = "exact_match"
+        ...
+        ...     def compute(self, *, prediction, references, metadata=None):
+        ...         # prediction is already extracted - no parsing needed
+        ...         is_correct = any(prediction == ref for ref in references)
+        ...         return MetricScore(
+        ...             metric_name=self.name,
+        ...             value=1.0 if is_correct else 0.0
+        ...         )
+    """
+
     name: str
     requires_reference: bool = True
 
@@ -72,6 +128,36 @@ class Metric(ABC):
         references: Sequence[Any],
         metadata: dict[str, Any] | None = None,
     ) -> entities.MetricScore:  # pragma: no cover - abstract
+        """Compute metric score.
+
+        Args:
+            prediction: Extracted prediction from model output (already processed
+                by extractor - do NOT re-extract or parse). Type depends on the
+                extractor used in the pipeline.
+            references: List of reference values in normalized format. Each element
+                can be:
+                - A scalar value (str, int, float, bool)
+                - A dict (for multi-value references like {"target": 122, "numbers": [...]})
+                - Any other type from the original reference
+            metadata: Optional metadata dict containing:
+                - "sample_id": Sample identifier (if available)
+                - Additional task-specific metadata
+
+        Returns:
+            MetricScore with computed value and optional details
+
+        Note:
+            The prediction parameter is already extracted by the pipeline's extractor.
+            Metrics should work with the extracted value directly, not attempt to
+            parse or extract again from raw output.
+
+        Example:
+            >>> def compute(self, *, prediction, references, metadata=None):
+            ...     # prediction is already extracted (e.g., "42", not "<answer>42</answer>")
+            ...     # references is a list (e.g., ["42"] or [{"target": 42, "numbers": [...]}])
+            ...     score_value = self._compare(prediction, references)
+            ...     return MetricScore(metric_name=self.name, value=score_value)
+        """
         raise NotImplementedError
 
 

themis/presets/__init__.py

@@ -0,0 +1,10 @@
+"""Preset configurations for common benchmarks and models.
+
+This module provides automatic configuration for popular benchmarks,
+eliminating the need for manual setup of prompts, metrics, and extractors.
+"""
+
+from themis.presets.benchmarks import get_benchmark_preset, list_benchmarks
+from themis.presets.models import parse_model_name
+
+__all__ = ["get_benchmark_preset", "list_benchmarks", "parse_model_name"]

themis/presets/benchmarks.py

@@ -0,0 +1,354 @@
+"""Benchmark preset configurations.
+
+This module provides pre-configured settings for popular benchmarks,
+including prompts, metrics, extractors, and data loaders.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, Callable, Sequence
+
+from themis.generation.templates import PromptTemplate
+from themis.interfaces import Extractor, Metric
+
+
+@dataclass
+class BenchmarkPreset:
+    """Configuration preset for a benchmark.
+    
+    Attributes:
+        name: Benchmark name
+        prompt_template: Default prompt template
+        metrics: List of metric instances
+        extractor: Output extractor
+        dataset_loader: Function to load the dataset
+        metadata_fields: Fields to include in task metadata
+        reference_field: Field containing the reference answer
+        dataset_id_field: Field containing the sample ID
+        description: Human-readable description
+    """
+    
+    name: str
+    prompt_template: PromptTemplate
+    metrics: list[Metric]
+    extractor: Extractor
+    dataset_loader: Callable[[int | None], Sequence[dict[str, Any]]]
+    metadata_fields: tuple[str, ...] = field(default_factory=tuple)
+    reference_field: str = "answer"
+    dataset_id_field: str = "id"
+    description: str = ""
+    
+    def load_dataset(self, limit: int | None = None) -> Sequence[dict[str, Any]]:
+        """Load the benchmark dataset.
+        
+        Args:
+            limit: Maximum number of samples to load
+        
+        Returns:
+            List of dataset samples
+        """
+        return self.dataset_loader(limit)
+
+
+# Registry of benchmark presets
+_BENCHMARK_REGISTRY: dict[str, BenchmarkPreset] = {}
+_REGISTRY_INITIALIZED = False
+
+
+def _ensure_registry_initialized() -> None:
+    """Initialize benchmark registry on first use (lazy loading)."""
+    global _REGISTRY_INITIALIZED
+    if not _REGISTRY_INITIALIZED:
+        _register_all_benchmarks()
+        _REGISTRY_INITIALIZED = True
+
+
+def register_benchmark(preset: BenchmarkPreset) -> None:
+    """Register a benchmark preset.
+    
+    Args:
+        preset: Benchmark preset configuration
+    """
+    _BENCHMARK_REGISTRY[preset.name.lower()] = preset
+
+
+def get_benchmark_preset(name: str) -> BenchmarkPreset:
+    """Get a benchmark preset by name.
+    
+    Args:
+        name: Benchmark name (case-insensitive)
+    
+    Returns:
+        Benchmark preset
+    
+    Raises:
+        ValueError: If benchmark is not found
+    """
+    _ensure_registry_initialized()
+    
+    name_lower = name.lower()
+    if name_lower not in _BENCHMARK_REGISTRY:
+        available = ", ".join(sorted(_BENCHMARK_REGISTRY.keys()))
+        raise ValueError(
+            f"Unknown benchmark: {name}. "
+            f"Available benchmarks: {available}"
+        )
+    return _BENCHMARK_REGISTRY[name_lower]
+
+
+def list_benchmarks() -> list[str]:
+    """List all registered benchmark names.
+    
+    Returns:
+        Sorted list of benchmark names
+    """
+    _ensure_registry_initialized()
+    return sorted(_BENCHMARK_REGISTRY.keys())
+
+
+# ============================================================================
+# Math Benchmarks
+# ============================================================================
+
+def _create_math500_preset() -> BenchmarkPreset:
+    """Create MATH-500 benchmark preset."""
+    from themis.datasets.math500 import load_math500 as load_math500_dataset
+    from themis.evaluation.extractors.math_verify_extractor import MathVerifyExtractor
+    from themis.evaluation.metrics.math_verify_accuracy import MathVerifyAccuracy
+    
+    def load_math500(limit: int | None = None) -> Sequence[dict[str, Any]]:
+        samples = load_math500_dataset(source="huggingface", limit=limit)
+        # Convert MathSample objects to dicts
+        return [s.to_generation_example() if hasattr(s, 'to_generation_example') else dict(s) for s in samples]
+    
+    prompt_template = PromptTemplate(
+        name="math500-zero-shot",
+        template=(
+            "Solve the following math problem step by step. "
+            "Put your final answer in \\boxed{{}}.\n\n"
+            "Problem: {problem}\n\n"
+            "Solution:"
+        ),
+    )
+    
+    return BenchmarkPreset(
+        name="math500",
+        prompt_template=prompt_template,
+        metrics=[MathVerifyAccuracy()],
+        extractor=MathVerifyExtractor(),
+        dataset_loader=load_math500,
+        metadata_fields=("subject", "level"),
+        reference_field="solution",
+        dataset_id_field="unique_id",
+        description="MATH-500 dataset with 500 competition math problems",
+    )
+
+
+def _create_gsm8k_preset() -> BenchmarkPreset:
+    """Create GSM8K benchmark preset."""
+    from themis.datasets.gsm8k import load_gsm8k as load_gsm8k_dataset
+    from themis.evaluation.extractors.math_verify_extractor import MathVerifyExtractor
+    from themis.evaluation.metrics.math_verify_accuracy import MathVerifyAccuracy
+    
+    def load_gsm8k(limit: int | None = None) -> Sequence[dict[str, Any]]:
+        samples = load_gsm8k_dataset(source="huggingface", split="test", limit=limit)
+        # Convert sample objects to dicts if needed
+        return [dict(s) if not isinstance(s, dict) else s for s in samples]
+    
+    prompt_template = PromptTemplate(
+        name="gsm8k-zero-shot",
+        template=(
+            "Solve this math problem step by step.\n\n"
+            "Q: {question}\n"
+            "A:"
+        ),
+    )
+    
+    return BenchmarkPreset(
+        name="gsm8k",
+        prompt_template=prompt_template,
+        metrics=[MathVerifyAccuracy()],
+        extractor=MathVerifyExtractor(),
+        dataset_loader=load_gsm8k,
+        metadata_fields=(),
+        reference_field="answer",
+        dataset_id_field="id",
+        description="GSM8K dataset with grade school math word problems",
+    )
+
+
+def _create_aime24_preset() -> BenchmarkPreset:
+    """Create AIME 2024 benchmark preset."""
+    from themis.datasets.competition_math import load_competition_math
+    from themis.evaluation.extractors.math_verify_extractor import MathVerifyExtractor
+    from themis.evaluation.metrics.math_verify_accuracy import MathVerifyAccuracy
+    
+    def load_aime24(limit: int | None = None) -> Sequence[dict[str, Any]]:
+        samples = load_competition_math(
+            dataset_id="aime24",
+            source="huggingface",
+            split="test",
+            limit=limit,
+        )
+        return [dict(s) if not isinstance(s, dict) else s for s in samples]
+    
+    prompt_template = PromptTemplate(
+        name="aime24-zero-shot",
+        template=(
+            "Solve the following AIME problem. "
+            "Your answer should be a number between 000 and 999.\n\n"
+            "Problem: {problem}\n\n"
+            "Solution:"
+        ),
+    )
+    
+    return BenchmarkPreset(
+        name="aime24",
+        prompt_template=prompt_template,
+        metrics=[MathVerifyAccuracy()],
+        extractor=MathVerifyExtractor(),
+        dataset_loader=load_aime24,
+        metadata_fields=("subject",),
+        reference_field="answer",
+        dataset_id_field="id",
+        description="AIME 2024 competition math problems",
+    )
+
+
+# ============================================================================
+# MCQ Benchmarks
+# ============================================================================
+
+def _create_mmlu_pro_preset() -> BenchmarkPreset:
+    """Create MMLU-Pro benchmark preset."""
+    from themis.datasets.mmlu_pro import load_mmlu_pro as load_mmlu_pro_dataset
+    from themis.evaluation.extractors.identity_extractor import IdentityExtractor
+    from themis.evaluation.metrics.exact_match import ExactMatch
+    
+    def load_mmlu_pro(limit: int | None = None) -> Sequence[dict[str, Any]]:
+        samples = load_mmlu_pro_dataset(source="huggingface", split="test", limit=limit)
+        return [dict(s) if not isinstance(s, dict) else s for s in samples]
+    
+    prompt_template = PromptTemplate(
+        name="mmlu-pro-zero-shot",
+        template=(
+            "Answer the following multiple choice question.\n\n"
+            "Question: {question}\n\n"
+            "Options:\n{options}\n\n"
+            "Answer:"
+        ),
+    )
+    
+    return BenchmarkPreset(
+        name="mmlu-pro",
+        prompt_template=prompt_template,
+        metrics=[ExactMatch()],
+        extractor=IdentityExtractor(),
+        dataset_loader=load_mmlu_pro,
+        metadata_fields=("category",),
+        reference_field="answer",
+        dataset_id_field="id",
+        description="MMLU-Pro professional-level multiple choice questions",
+    )
+
+
+def _create_supergpqa_preset() -> BenchmarkPreset:
+    """Create SuperGPQA benchmark preset."""
+    from themis.datasets.super_gpqa import load_super_gpqa as load_supergpqa_dataset
+    from themis.evaluation.extractors.identity_extractor import IdentityExtractor
+    from themis.evaluation.metrics.exact_match import ExactMatch
+    
+    def load_supergpqa(limit: int | None = None) -> Sequence[dict[str, Any]]:
+        samples = load_supergpqa_dataset(source="huggingface", split="test", limit=limit)
+        return [dict(s) if not isinstance(s, dict) else s for s in samples]
+    
+    prompt_template = PromptTemplate(
+        name="supergpqa-zero-shot",
+        template=(
+            "Answer the following science question.\n\n"
+            "Question: {question}\n\n"
+            "Choices:\n{choices}\n\n"
+            "Answer:"
+        ),
+    )
+    
+    return BenchmarkPreset(
+        name="supergpqa",
+        prompt_template=prompt_template,
+        metrics=[ExactMatch()],
+        extractor=IdentityExtractor(),
+        dataset_loader=load_supergpqa,
+        metadata_fields=("subject",),
+        reference_field="answer",
+        dataset_id_field="id",
+        description="SuperGPQA graduate-level science questions",
+    )
+
+
+# ============================================================================
+# Demo/Test Benchmarks
+# ============================================================================
+
+def _create_demo_preset() -> BenchmarkPreset:
+    """Create demo benchmark preset for testing."""
+    from themis.evaluation.extractors.identity_extractor import IdentityExtractor
+    from themis.evaluation.metrics.exact_match import ExactMatch
+    
+    def load_demo(limit: int | None = None) -> Sequence[dict[str, Any]]:
+        samples = [
+            {"id": "demo-1", "question": "What is 2 + 2?", "answer": "4"},
+            {"id": "demo-2", "question": "What is the capital of France?", "answer": "Paris"},
+            {"id": "demo-3", "question": "What is 10 * 5?", "answer": "50"},
+        ]
+        if limit is not None:
+            samples = samples[:limit]
+        return samples
+    
+    prompt_template = PromptTemplate(
+        name="demo",
+        template="Q: {question}\nA:",
+    )
+    
+    return BenchmarkPreset(
+        name="demo",
+        prompt_template=prompt_template,
+        metrics=[ExactMatch()],
+        extractor=IdentityExtractor(),
+        dataset_loader=load_demo,
+        metadata_fields=(),
+        reference_field="answer",
+        dataset_id_field="id",
+        description="Demo benchmark for testing",
+    )
+
+
+# ============================================================================
+# Register all benchmarks (lazy initialization)
+# ============================================================================
+
+def _register_all_benchmarks() -> None:
+    """Register all built-in benchmarks.
+    
+    This is called lazily on first use to avoid importing heavy dependencies
+    (datasets, models, etc.) until actually needed.
+    """
+    # Math benchmarks
+    register_benchmark(_create_math500_preset())
+    register_benchmark(_create_gsm8k_preset())
+    register_benchmark(_create_aime24_preset())
+    
+    # MCQ benchmarks
+    register_benchmark(_create_mmlu_pro_preset())
+    register_benchmark(_create_supergpqa_preset())
+    
+    # Demo
+    register_benchmark(_create_demo_preset())
+
+
+__all__ = [
+    "BenchmarkPreset",
+    "register_benchmark",
+    "get_benchmark_preset",
+    "list_benchmarks",
+]