themis-eval 0.1.0__py3-none-any.whl → 0.2.0__py3-none-any.whl

themis/evaluation/metrics/code/pass_at_k.py

@@ -0,0 +1,181 @@
+"""Pass@k metric for code generation evaluation.
+
+Pass@k measures functional correctness by executing k generated code samples
+and checking if any of them pass the test cases.
+
+References:
+    Chen et al. (2021). Evaluating Large Language Models Trained on Code.
+    (HumanEval paper)
+"""
+
+from __future__ import annotations
+
+import math
+from typing import Any, Sequence
+
+from themis.core.entities import MetricScore
+from themis.interfaces import Metric
+
+
+def estimate_pass_at_k(n: int, c: int, k: int) -> float:
+    """Estimate pass@k using unbiased estimator.
+    
+    This is the standard estimator from the HumanEval paper.
+    
+    Args:
+        n: Total number of samples generated
+        c: Number of samples that passed
+        k: k value for pass@k
+    
+    Returns:
+        Estimated pass@k probability
+    
+    Example:
+        >>> # Generated 10 samples, 3 passed, compute pass@1
+        >>> estimate_pass_at_k(n=10, c=3, k=1)
+        0.3
+        
+        >>> # Generated 100 samples, 30 passed, compute pass@10
+        >>> estimate_pass_at_k(n=100, c=30, k=10)
+        0.8926
+    """
+    if n - c < k:
+        return 1.0
+    
+    # Unbiased estimator: 1 - C(n-c, k) / C(n, k)
+    # = 1 - product((n-c-i)/(n-i) for i in range(k))
+    result = 1.0
+    for i in range(k):
+        result *= (n - c - i) / (n - i)
+    
+    return 1.0 - result
+
+
+class PassAtK(Metric):
+    """Pass@k metric for code generation.
+    
+    Pass@k measures the probability that at least one of k generated samples
+    passes all test cases. It's the standard metric for evaluating code
+    generation models like Codex, CodeGen, etc.
+    
+    The metric requires:
+    - Multiple samples per problem (num_samples >= k)
+    - Test cases to execute against
+    - Safe code execution environment
+    
+    Attributes:
+        name: Metric identifier ("pass_at_k")
+        k: Number of samples to consider
+        timeout: Maximum execution time per sample (seconds)
+        require_all_tests: Whether all tests must pass (vs any test)
+    
+    Example:
+        >>> from themis.evaluation.metrics.code import PassAtK
+        >>> metric = PassAtK(k=1)
+        >>> score = metric.compute(
+        ...     prediction={
+        ...         "samples": ["def add(a, b): return a + b", ...],
+        ...         "test_results": [True, False, ...],
+        ...     },
+        ...     references=[]
+        ... )
+        >>> print(f"Pass@1: {score.value:.2%}")
+        Pass@1: 30.00%
+    """
+    
+    requires_reference = False  # Uses test execution, not reference matching
+    
+    def __init__(
+        self,
+        k: int = 1,
+        timeout: float = 3.0,
+        require_all_tests: bool = True,
+    ):
+        """Initialize Pass@k metric.
+        
+        Args:
+            k: Number of samples for pass@k estimation
+            timeout: Maximum execution time per sample (seconds)
+            require_all_tests: Whether all test cases must pass (default: True)
+        """
+        self.name = f"pass_at_{k}"
+        self.k = k
+        self.timeout = timeout
+        self.require_all_tests = require_all_tests
+    
+    def compute(
+        self,
+        *,
+        prediction: Any,
+        references: Sequence[Any],
+        metadata: dict[str, Any] | None = None,
+    ) -> MetricScore:
+        """Compute Pass@k score.
+        
+        Args:
+            prediction: Dictionary containing:
+                - "samples": List of generated code samples
+                - "test_results": List of booleans (True if passed)
+                - "execution_errors": Optional list of error messages
+            references: Not used (test-based evaluation)
+            metadata: Optional metadata dict
+        
+        Returns:
+            MetricScore with estimated pass@k probability
+        
+        Note:
+            The prediction should be prepared by ExecutionAccuracy metric
+            or similar execution framework.
+        """
+        if not isinstance(prediction, dict):
+            return MetricScore(
+                metric_name=self.name,
+                value=0.0,
+                details={"error": "Prediction must be dict with samples and test_results"},
+                metadata=metadata or {},
+            )
+        
+        samples = prediction.get("samples", [])
+        test_results = prediction.get("test_results", [])
+        
+        if not samples or not test_results:
+            return MetricScore(
+                metric_name=self.name,
+                value=0.0,
+                details={
+                    "error": "Missing samples or test_results",
+                    "num_samples": len(samples),
+                    "num_results": len(test_results),
+                },
+                metadata=metadata or {},
+            )
+        
+        # Count number of samples and passes
+        n = len(test_results)
+        c = sum(1 for result in test_results if result)
+        
+        # Estimate pass@k
+        if n < self.k:
+            # Not enough samples, use empirical rate
+            pass_at_k = c / n if n > 0 else 0.0
+            warning = f"Only {n} samples available for pass@{self.k}"
+        else:
+            pass_at_k = estimate_pass_at_k(n, c, self.k)
+            warning = None
+        
+        return MetricScore(
+            metric_name=self.name,
+            value=pass_at_k,
+            details={
+                "k": self.k,
+                "n_samples": n,
+                "n_passed": c,
+                "pass_rate": c / n if n > 0 else 0.0,
+                "pass_at_k": pass_at_k,
+                "warning": warning,
+            },
+            metadata=metadata or {},
+        )
+
+
+__all__ = ["PassAtK", "estimate_pass_at_k"]

themis/evaluation/metrics/composite_metric.py

@@ -0,0 +1,47 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any, Sequence
+
+from themis.core import entities as core_entities
+from themis.interfaces import Metric as MetricInterface
+
+
+@dataclass
+class CompositeMetric(MetricInterface):
+    children: Sequence[MetricInterface]
+
+    def __post_init__(self) -> None:
+        self.name = "CompositeMetric"
+        self.requires_reference = any(
+            getattr(child, "requires_reference", True) for child in self.children
+        )
+
+    def compute(
+        self,
+        *,
+        prediction: Any,
+        references: Sequence[Any],
+        metadata: dict[str, Any] | None = None,
+    ) -> core_entities.MetricScore:
+        child_results = [
+            child.compute(
+                prediction=prediction, references=references, metadata=metadata
+            )
+            for child in self.children
+        ]
+        if not child_results:
+            return core_entities.MetricScore(
+                metric_name=self.name,
+                value=0.0,
+                details={},
+                metadata=dict(metadata or {}),
+            )
+        value = sum(result.value for result in child_results) / len(child_results)
+        details = {result.metric_name: result.details for result in child_results}
+        return core_entities.MetricScore(
+            metric_name=self.name,
+            value=value,
+            details=details,
+            metadata=dict(metadata or {}),
+        )

themis/evaluation/metrics/consistency_metric.py

@@ -0,0 +1,80 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any, Sequence
+
+from themis.core import entities as core_entities
+from themis.interfaces import Metric as MetricInterface
+
+
+def _normalize_text(value: str, case_sensitive: bool, strip_whitespace: bool) -> str:
+    if strip_whitespace:
+        value = value.strip()
+    if not case_sensitive:
+        value = value.lower()
+    return value
+
+
+@dataclass
+class ConsistencyMetric(MetricInterface):
+    case_sensitive: bool = False
+    strip_whitespace: bool = True
+
+    def __post_init__(self) -> None:
+        self.name = "Consistency"
+        self.requires_reference = False
+
+    def compute(
+        self,
+        *,
+        prediction: Any,
+        references: Sequence[Any],
+        metadata: dict[str, Any] | None = None,
+    ) -> core_entities.MetricScore:
+        md = dict(metadata or {})
+
+        outputs: list[str]
+        if isinstance(prediction, (list, tuple)):
+            outputs = [str(p) for p in prediction]
+        else:
+            outputs = [str(prediction)]
+
+        normalized = [
+            _normalize_text(text, self.case_sensitive, self.strip_whitespace)
+            for text in outputs
+        ]
+
+        majority_correct = None
+        reference_text = None
+        if references:
+            reference_text = _normalize_text(
+                str(references[0]), self.case_sensitive, self.strip_whitespace
+            )
+            correct = [1.0 if out == reference_text else 0.0 for out in normalized]
+            majority_correct = sum(correct) / max(1, len(correct))
+
+        from collections import Counter
+
+        counter = Counter(normalized)
+        mode_count = max(counter.values()) if counter else 0
+        agreement = mode_count / max(1, len(normalized))
+
+        flips = 0
+        for i in range(1, len(normalized)):
+            if normalized[i] != normalized[i - 1]:
+                flips += 1
+        flip_rate = flips / max(1, len(normalized) - 1)
+
+        value = majority_correct if majority_correct is not None else agreement
+
+        return core_entities.MetricScore(
+            metric_name=self.name,
+            value=float(value),
+            details={
+                "agreement": agreement,
+                "flip_rate": flip_rate,
+                "outputs": outputs,
+                "reference": reference_text,
+            },
+            metadata=md,
+        )

themis/evaluation/metrics/exact_match.py

@@ -0,0 +1,51 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any, Sequence
+
+from themis.core import entities as core_entities
+from themis.interfaces import Metric as MetricInterface
+
+
+def _normalize_text(value: str, case_sensitive: bool, strip_whitespace: bool) -> str:
+    if strip_whitespace:
+        value = value.strip()
+    if not case_sensitive:
+        value = value.lower()
+    return value
+
+
+@dataclass
+class ExactMatch(MetricInterface):
+    case_sensitive: bool = False
+    strip_whitespace: bool = True
+
+    def __post_init__(self) -> None:
+        self.name = "ExactMatch"
+
+    def compute(
+        self,
+        *,
+        prediction: Any,
+        references: Sequence[Any],
+        metadata: dict[str, Any] | None = None,
+    ) -> core_entities.MetricScore:
+        metadata = dict(metadata or {})
+        normalized_prediction = _normalize_text(
+            str(prediction), self.case_sensitive, self.strip_whitespace
+        )
+        matched_reference: str | None = None
+        for reference in references:
+            normalized_reference = _normalize_text(
+                str(reference), self.case_sensitive, self.strip_whitespace
+            )
+            if normalized_prediction == normalized_reference:
+                matched_reference = str(reference)
+                break
+        value = 1.0 if matched_reference is not None else 0.0
+        return core_entities.MetricScore(
+            metric_name=self.name,
+            value=value,
+            details={"matched_reference": matched_reference},
+            metadata=metadata,
+        )

themis/evaluation/metrics/length_difference_tolerance.py

@@ -0,0 +1,33 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any, Sequence
+
+from themis.core import entities as core_entities
+from themis.interfaces import Metric as MetricInterface
+
+
+@dataclass
+class LengthDifferenceTolerance(MetricInterface):
+    max_delta: int = 0
+
+    def __post_init__(self) -> None:
+        self.name = "LengthDifferenceTolerance"
+
+    def compute(
+        self,
+        *,
+        prediction: Any,
+        references: Sequence[Any],
+        metadata: dict[str, Any] | None = None,
+    ) -> core_entities.MetricScore:
+        metadata = dict(metadata or {})
+        reference = str(references[0]) if references else ""
+        delta = abs(len(str(prediction)) - len(reference))
+        value = 1.0 if delta <= self.max_delta else 0.0
+        return core_entities.MetricScore(
+            metric_name=self.name,
+            value=value,
+            details={"delta": delta},
+            metadata=metadata,
+        )

themis/evaluation/metrics/math_verify_accuracy.py

@@ -0,0 +1,40 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any, Sequence
+
+from themis.core import entities as core_entities
+from themis.evaluation import math_verify_utils
+from themis.interfaces import Metric as MetricInterface
+
+
+@dataclass
+class MathVerifyAccuracy(MetricInterface):
+    """Numeric equivalence check using math-verify."""
+
+    def __post_init__(self) -> None:
+        math_verify_utils.require_math_verify()
+        self.name = "MathVerifyAccuracy"
+
+    def compute(
+        self,
+        *,
+        prediction: Any,
+        references: Sequence[Any],
+        metadata: dict[str, Any] | None = None,
+    ) -> core_entities.MetricScore:
+        math_verify_utils.require_math_verify()
+        metadata = dict(metadata or {})
+        prediction_expr = math_verify_utils.parse_expression(str(prediction))
+        passed = False
+        for reference in references:
+            reference_expr = math_verify_utils.parse_expression(str(reference))
+            if math_verify_utils.verify_expressions(reference_expr, prediction_expr):
+                passed = True
+                break
+        return core_entities.MetricScore(
+            metric_name=self.name,
+            value=1.0 if passed else 0.0,
+            details={"verified": passed},
+            metadata=metadata,
+        )

themis/evaluation/metrics/nlp/__init__.py

@@ -0,0 +1,21 @@
+"""NLP evaluation metrics.
+
+This module provides standard NLP metrics for text generation evaluation:
+- BLEU: Bilingual Evaluation Understudy for translation quality
+- ROUGE: Recall-Oriented Understudy for Gisting Evaluation for summarization
+- BERTScore: Contextual embeddings-based evaluation
+- METEOR: Metric for Evaluation of Translation with Explicit ORdering
+"""
+
+from themis.evaluation.metrics.nlp.bleu import BLEU
+from themis.evaluation.metrics.nlp.rouge import ROUGE, ROUGEVariant
+from themis.evaluation.metrics.nlp.bertscore import BERTScore
+from themis.evaluation.metrics.nlp.meteor import METEOR
+
+__all__ = [
+    "BLEU",
+    "ROUGE",
+    "ROUGEVariant",
+    "BERTScore",
+    "METEOR",
+]

themis/evaluation/metrics/nlp/bertscore.py

@@ -0,0 +1,138 @@
+"""BERTScore metric implementation.
+
+BERTScore computes similarity using contextual embeddings from BERT-like models
+instead of exact word matches.
+
+References:
+    Zhang et al. (2020). BERTScore: Evaluating Text Generation with BERT.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Sequence
+
+from themis.core.entities import MetricScore
+from themis.interfaces import Metric
+
+
+class BERTScore(Metric):
+    """BERTScore metric using bert-score library.
+    
+    BERTScore leverages contextual embeddings from pre-trained models (BERT, RoBERTa, etc.)
+    to compute semantic similarity between generated and reference texts. It's more
+    robust to paraphrasing than exact n-gram matching methods.
+    
+    The metric computes token-level cosine similarity between embeddings and aggregates
+    using precision, recall, and F1.
+    
+    Attributes:
+        name: Metric identifier ("bertscore")
+        model_type: Pre-trained model to use for embeddings
+        lang: Language code for automatic model selection
+        rescale_with_baseline: Whether to rescale scores using baseline
+    
+    Example:
+        >>> from themis.evaluation.metrics.nlp import BERTScore
+        >>> metric = BERTScore(model_type="microsoft/deberta-xlarge-mnli")
+        >>> score = metric.compute(
+        ...     prediction="The cat sat on the mat",
+        ...     references=["A cat is sitting on a mat"]
+        ... )
+        >>> print(f"BERTScore F1: {score.value:.4f}")
+        BERTScore F1: 0.9234
+    """
+    
+    requires_reference = True
+    
+    def __init__(
+        self,
+        model_type: str | None = None,
+        lang: str | None = None,
+        rescale_with_baseline: bool = True,
+        device: str | None = None,
+    ):
+        """Initialize BERTScore metric.
+        
+        Args:
+            model_type: Pre-trained model identifier. Popular choices:
+                - "microsoft/deberta-xlarge-mnli" (recommended, large)
+                - "microsoft/deberta-large-mnli" (good balance)
+                - "roberta-large" (fast, good quality)
+                - "bert-base-uncased" (fastest, lower quality)
+            lang: Language code (e.g., "en", "zh", "fr"). If provided,
+                automatically selects appropriate model.
+            rescale_with_baseline: Whether to rescale scores using baseline
+                (recommended for human correlation)
+            device: Device to use ("cuda", "cpu", or None for auto-detect)
+        """
+        self.name = "bertscore"
+        self.model_type = model_type
+        self.lang = lang
+        self.rescale_with_baseline = rescale_with_baseline
+        self.device = device
+        
+        # Lazy import bert-score (not required for all users)
+        try:
+            import bert_score
+            self._bert_score = bert_score
+        except ImportError:
+            raise ImportError(
+                "bert-score is required for BERTScore metric. "
+                "Install it with: pip install bert-score"
+            )
+    
+    def compute(
+        self,
+        *,
+        prediction: Any,
+        references: Sequence[Any],
+        metadata: dict[str, Any] | None = None,
+    ) -> MetricScore:
+        """Compute BERTScore.
+        
+        Args:
+            prediction: Generated text (already extracted by pipeline)
+            references: List of reference texts
+            metadata: Optional metadata dict
+        
+        Returns:
+            MetricScore with BERTScore F1 and precision/recall details
+        """
+        # Convert to strings
+        pred_str = str(prediction)
+        ref_strs = [str(ref) for ref in references]
+        
+        # Compute BERTScore
+        # Note: bert_score.score expects lists of predictions and references
+        P, R, F1 = self._bert_score.score(
+            [pred_str] * len(ref_strs),  # Repeat prediction for each reference
+            ref_strs,
+            model_type=self.model_type,
+            lang=self.lang,
+            rescale_with_baseline=self.rescale_with_baseline,
+            device=self.device,
+            verbose=False,
+        )
+        
+        # Take maximum F1 across references
+        max_idx = F1.argmax().item()
+        max_precision = P[max_idx].item()
+        max_recall = R[max_idx].item()
+        max_f1 = F1[max_idx].item()
+        
+        return MetricScore(
+            metric_name=self.name,
+            value=max_f1,  # Use F1 as primary score
+            details={
+                "precision": max_precision,
+                "recall": max_recall,
+                "f1": max_f1,
+                "model_type": self.model_type or f"auto-{self.lang}",
+                "num_references": len(ref_strs),
+                "rescaled": self.rescale_with_baseline,
+            },
+            metadata=metadata or {},
+        )
+
+
+__all__ = ["BERTScore"]