themis-eval 0.1.1__py3-none-any.whl → 0.2.1__py3-none-any.whl

themis/evaluation/metrics/nlp/bleu.py

@@ -0,0 +1,129 @@
+"""BLEU (Bilingual Evaluation Understudy) metric implementation.
+
+BLEU measures the similarity between generated text and reference translations
+using n-gram precision with brevity penalty.
+
+References:
+    Papineni et al. (2002). BLEU: a Method for Automatic Evaluation of Machine Translation.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Sequence
+
+from themis.core.entities import MetricScore
+from themis.interfaces import Metric
+
+
+class BLEU(Metric):
+    """BLEU metric using sacrebleu library.
+    
+    BLEU is a precision-based metric that computes n-gram overlap between
+    generated text and reference translations. It includes a brevity penalty
+    to penalize short translations.
+    
+    Attributes:
+        name: Metric identifier ("bleu")
+        lowercase: Whether to lowercase text before scoring
+        tokenize: Tokenization method ("13a", "intl", "zh", "ja-mecab", etc.)
+        max_ngram_order: Maximum n-gram order (default: 4)
+    
+    Example:
+        >>> from themis.evaluation.metrics.nlp import BLEU
+        >>> metric = BLEU()
+        >>> score = metric.compute(
+        ...     prediction="The cat sat on the mat",
+        ...     references=["The cat is on the mat", "A cat is sitting on a mat"]
+        ... )
+        >>> print(f"BLEU: {score.value:.4f}")
+        BLEU: 0.4523
+    """
+    
+    requires_reference = True
+    
+    def __init__(
+        self,
+        lowercase: bool = False,
+        tokenize: str = "13a",
+        max_ngram_order: int = 4,
+    ):
+        """Initialize BLEU metric.
+        
+        Args:
+            lowercase: Convert text to lowercase before scoring
+            tokenize: Tokenization method:
+                - "13a": Default Moses tokenizer (punctuation split)
+                - "intl": International tokenizer
+                - "zh": Chinese tokenizer
+                - "ja-mecab": Japanese MeCab tokenizer
+                - "none": No tokenization
+            max_ngram_order: Maximum n-gram order (typically 4)
+        """
+        self.name = "bleu"
+        self.lowercase = lowercase
+        self.tokenize = tokenize
+        self.max_ngram_order = max_ngram_order
+        
+        # Lazy import sacrebleu (not required for all users)
+        try:
+            from sacrebleu import BLEU as SacreBLEU
+            self._scorer = SacreBLEU(
+                lowercase=lowercase,
+                tokenize=tokenize,
+                max_ngram_order=max_ngram_order,
+            )
+        except ImportError:
+            raise ImportError(
+                "sacrebleu is required for BLEU metric. "
+                "Install it with: pip install sacrebleu"
+            )
+    
+    def compute(
+        self,
+        *,
+        prediction: Any,
+        references: Sequence[Any],
+        metadata: dict[str, Any] | None = None,
+    ) -> MetricScore:
+        """Compute BLEU score.
+        
+        Args:
+            prediction: Generated text (already extracted by pipeline)
+            references: List of reference translations
+            metadata: Optional metadata dict
+        
+        Returns:
+            MetricScore with BLEU value (0.0-1.0) and detailed scores
+        """
+        # Convert to strings
+        pred_str = str(prediction)
+        ref_strs = [str(ref) for ref in references]
+        
+        # Compute BLEU score
+        score_obj = self._scorer.sentence_score(pred_str, ref_strs)
+        
+        # Extract scores (sacrebleu returns 0-100, we normalize to 0-1)
+        bleu_score = score_obj.score / 100.0
+        
+        # Extract precision scores for each n-gram
+        precisions = [p / 100.0 for p in score_obj.precisions]
+        
+        return MetricScore(
+            metric_name=self.name,
+            value=bleu_score,
+            details={
+                "bleu_score": bleu_score,
+                "precision_1": precisions[0] if len(precisions) > 0 else 0.0,
+                "precision_2": precisions[1] if len(precisions) > 1 else 0.0,
+                "precision_3": precisions[2] if len(precisions) > 2 else 0.0,
+                "precision_4": precisions[3] if len(precisions) > 3 else 0.0,
+                "brevity_penalty": score_obj.bp,
+                "length_ratio": score_obj.sys_len / score_obj.ref_len if score_obj.ref_len > 0 else 0.0,
+                "sys_len": score_obj.sys_len,
+                "ref_len": score_obj.ref_len,
+            },
+            metadata=metadata or {},
+        )
+
+
+__all__ = ["BLEU"]

themis/evaluation/metrics/nlp/meteor.py

@@ -0,0 +1,153 @@
+"""METEOR (Metric for Evaluation of Translation with Explicit ORdering) metric.
+
+METEOR is an MT evaluation metric that addresses some weaknesses of BLEU by
+incorporating stemming, synonymy, and explicit word ordering.
+
+References:
+    Banerjee & Lavie (2005). METEOR: An Automatic Metric for MT Evaluation
+    with Improved Correlation with Human Judgments.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Sequence
+
+from themis.core.entities import MetricScore
+from themis.interfaces import Metric
+
+
+class METEOR(Metric):
+    """METEOR metric using nltk library.
+    
+    METEOR compares generated text to references using:
+    - Exact word matching
+    - Stemming (using Porter stemmer)
+    - Synonymy (using WordNet)
+    - Word order (using chunk matching)
+    
+    It computes a weighted F-score with emphasis on recall and applies a penalty
+    for word order differences.
+    
+    Attributes:
+        name: Metric identifier ("meteor")
+        alpha: Weight for precision vs recall (default: 0.9, favors recall)
+        beta: Weight for fragmentation penalty (default: 3.0)
+        gamma: Fragmentation penalty coefficient (default: 0.5)
+    
+    Example:
+        >>> from themis.evaluation.metrics.nlp import METEOR
+        >>> metric = METEOR()
+        >>> score = metric.compute(
+        ...     prediction="The cat sat on the mat",
+        ...     references=["The cat is on the mat", "A cat sits on a mat"]
+        ... )
+        >>> print(f"METEOR: {score.value:.4f}")
+        METEOR: 0.8234
+    """
+    
+    requires_reference = True
+    
+    def __init__(
+        self,
+        alpha: float = 0.9,
+        beta: float = 3.0,
+        gamma: float = 0.5,
+    ):
+        """Initialize METEOR metric.
+        
+        Args:
+            alpha: Weight for precision vs recall (0-1). Higher values favor recall.
+                Default 0.9 emphasizes recall like original METEOR.
+            beta: Weight for fragmentation penalty (typically 3.0)
+            gamma: Fragmentation penalty coefficient (typically 0.5)
+        """
+        self.name = "meteor"
+        self.alpha = alpha
+        self.beta = beta
+        self.gamma = gamma
+        
+        # Lazy import nltk (not required for all users)
+        try:
+            from nltk.translate import meteor_score as meteor
+            self._meteor = meteor
+            
+            # Download required NLTK data if not present
+            import nltk
+            try:
+                nltk.data.find('corpora/wordnet')
+            except LookupError:
+                print("Downloading WordNet data for METEOR...")
+                nltk.download('wordnet', quiet=True)
+            
+            try:
+                nltk.data.find('omw-1.4')
+            except LookupError:
+                print("Downloading OMW data for METEOR...")
+                nltk.download('omw-1.4', quiet=True)
+                
+        except ImportError:
+            raise ImportError(
+                "nltk is required for METEOR metric. "
+                "Install it with: pip install nltk"
+            )
+    
+    def compute(
+        self,
+        *,
+        prediction: Any,
+        references: Sequence[Any],
+        metadata: dict[str, Any] | None = None,
+    ) -> MetricScore:
+        """Compute METEOR score.
+        
+        Args:
+            prediction: Generated text (already extracted by pipeline)
+            references: List of reference texts
+            metadata: Optional metadata dict
+        
+        Returns:
+            MetricScore with METEOR value (0.0-1.0)
+        """
+        # Convert to strings and tokenize
+        pred_str = str(prediction)
+        ref_strs = [str(ref) for ref in references]
+        
+        # Tokenize (simple whitespace tokenization)
+        pred_tokens = pred_str.split()
+        ref_tokens_list = [ref.split() for ref in ref_strs]
+        
+        # Compute METEOR score
+        # Note: nltk's meteor_score takes one reference at a time
+        # We compute for each reference and take the maximum
+        max_score = 0.0
+        
+        for ref_tokens in ref_tokens_list:
+            try:
+                score = self._meteor.meteor_score(
+                    [ref_tokens],  # References should be list of tokenized references
+                    pred_tokens,   # Hypothesis is tokenized prediction
+                    alpha=self.alpha,
+                    beta=self.beta,
+                    gamma=self.gamma,
+                )
+                max_score = max(max_score, score)
+            except Exception as e:
+                # Handle edge cases (empty strings, etc.)
+                print(f"Warning: METEOR computation failed: {e}")
+                continue
+        
+        return MetricScore(
+            metric_name=self.name,
+            value=max_score,
+            details={
+                "meteor_score": max_score,
+                "num_references": len(ref_strs),
+                "alpha": self.alpha,
+                "beta": self.beta,
+                "gamma": self.gamma,
+            },
+            metadata=metadata or {},
+        )
+
+
+__all__ = ["METEOR"]

themis/evaluation/metrics/nlp/rouge.py

@@ -0,0 +1,136 @@
+"""ROUGE (Recall-Oriented Understudy for Gisting Evaluation) metric.
+
+ROUGE measures overlap between generated text and reference summaries
+using n-grams and longest common subsequence.
+
+References:
+    Lin (2004). ROUGE: A Package for Automatic Evaluation of Summaries.
+"""
+
+from __future__ import annotations
+
+from enum import Enum
+from typing import Any, Sequence
+
+from themis.core.entities import MetricScore
+from themis.interfaces import Metric
+
+
+class ROUGEVariant(str, Enum):
+    """ROUGE metric variants."""
+    
+    ROUGE_1 = "rouge1"  # Unigram overlap
+    ROUGE_2 = "rouge2"  # Bigram overlap
+    ROUGE_L = "rougeL"  # Longest common subsequence
+    ROUGE_L_SUM = "rougeLsum"  # LCS with summary-level computation
+
+
+class ROUGE(Metric):
+    """ROUGE metric using rouge-score library.
+    
+    ROUGE is a recall-oriented metric that measures n-gram overlap between
+    generated text and reference summaries. It's commonly used for evaluating
+    text summarization and text generation tasks.
+    
+    Variants:
+        - ROUGE-1: Unigram overlap
+        - ROUGE-2: Bigram overlap
+        - ROUGE-L: Longest common subsequence (sentence-level)
+        - ROUGE-Lsum: Longest common subsequence (summary-level)
+    
+    Attributes:
+        name: Metric identifier (e.g., "rouge1", "rouge2", "rougeL")
+        variant: Which ROUGE variant to compute
+        use_stemmer: Whether to use Porter stemmer
+    
+    Example:
+        >>> from themis.evaluation.metrics.nlp import ROUGE, ROUGEVariant
+        >>> metric = ROUGE(variant=ROUGEVariant.ROUGE_2)
+        >>> score = metric.compute(
+        ...     prediction="The quick brown fox jumps over the lazy dog",
+        ...     references=["A quick brown fox jumped over a lazy dog"]
+        ... )
+        >>> print(f"ROUGE-2 F1: {score.value:.4f}")
+        ROUGE-2 F1: 0.6154
+    """
+    
+    requires_reference = True
+    
+    def __init__(
+        self,
+        variant: ROUGEVariant = ROUGEVariant.ROUGE_L,
+        use_stemmer: bool = True,
+    ):
+        """Initialize ROUGE metric.
+        
+        Args:
+            variant: Which ROUGE variant to compute
+            use_stemmer: Whether to use Porter stemmer for word matching
+        """
+        self.variant = variant
+        self.use_stemmer = use_stemmer
+        self.name = variant.value
+        
+        # Lazy import rouge-score (not required for all users)
+        try:
+            from rouge_score import rouge_scorer
+            self._scorer = rouge_scorer.RougeScorer(
+                [variant.value],
+                use_stemmer=use_stemmer,
+            )
+        except ImportError:
+            raise ImportError(
+                "rouge-score is required for ROUGE metric. "
+                "Install it with: pip install rouge-score"
+            )
+    
+    def compute(
+        self,
+        *,
+        prediction: Any,
+        references: Sequence[Any],
+        metadata: dict[str, Any] | None = None,
+    ) -> MetricScore:
+        """Compute ROUGE score.
+        
+        Args:
+            prediction: Generated text (already extracted by pipeline)
+            references: List of reference summaries
+            metadata: Optional metadata dict
+        
+        Returns:
+            MetricScore with ROUGE F1 score and precision/recall details
+        """
+        # Convert to strings
+        pred_str = str(prediction)
+        ref_strs = [str(ref) for ref in references]
+        
+        # Compute ROUGE for each reference and take the maximum
+        max_precision = 0.0
+        max_recall = 0.0
+        max_f1 = 0.0
+        
+        for ref_str in ref_strs:
+            scores = self._scorer.score(ref_str, pred_str)
+            rouge_score = scores[self.variant.value]
+            
+            if rouge_score.fmeasure > max_f1:
+                max_precision = rouge_score.precision
+                max_recall = rouge_score.recall
+                max_f1 = rouge_score.fmeasure
+        
+        return MetricScore(
+            metric_name=self.name,
+            value=max_f1,  # Use F1 as primary score
+            details={
+                "precision": max_precision,
+                "recall": max_recall,
+                "f1": max_f1,
+                "variant": self.variant.value,
+                "num_references": len(ref_strs),
+            },
+            metadata=metadata or {},
+        )
+
+
+__all__ = ["ROUGE", "ROUGEVariant"]

themis/evaluation/pipelines/standard_pipeline.py

@@ -4,6 +4,7 @@ from __future__ import annotations
 
 import logging
 import time
+import warnings
 from typing import Callable, Sequence
 
 from themis.core import entities as core_entities
@@ -35,19 +36,49 @@ def _default_reference_selector(record: core_entities.GenerationRecord):
     return reference.value
 
 
-def _normalize_references(reference):
-    """Normalize reference to list format.
+def _normalize_references(reference) -> list:
+    """Normalize reference to list format for metric consumption.
+
+    This function converts various reference formats into a standardized list
+    that metrics can reliably consume. The normalized format is always a list
+    where each element represents one reference value.
 
     Args:
-        reference: Reference value
+        reference: Reference value in various formats:
+            - Reference object: Extracts .value field
+            - dict: Kept as-is in a list (for multi-value references)
+            - list/tuple: Returned as list
+            - scalar: Wrapped in a list
 
     Returns:
-        List of references
+        List of reference values. Each element can be:
+        - A scalar value (str, int, float, bool)
+        - A dict (for multi-value references like {"target": 122, "numbers": [...]})
+        - Any other type from the original reference
+
+    Examples:
+        >>> _normalize_references(Reference(kind="answer", value="42"))
+        ["42"]
+
+        >>> _normalize_references(Reference(kind="task", value={"target": 122, "numbers": [25, 50]}))
+        [{"target": 122, "numbers": [25, 50]}]
+
+        >>> _normalize_references(["yes", "no", "maybe"])
+        ["yes", "no", "maybe"]
+
+        >>> _normalize_references("42")
+        ["42"]
+
+    Note:
+        Metrics receive references in this normalized format and should handle
+        both simple values and dict values appropriately.
     """
     if isinstance(reference, core_entities.Reference):
         reference = reference.value
     if isinstance(reference, list):
         return reference
+    if isinstance(reference, tuple):
+        return list(reference)
     return [reference]
 
 
@@ -89,12 +120,21 @@ class EvaluationPipeline:
         Args:
             extractor: Extractor for parsing model output
             metrics: List of metrics to compute
-            reference_selector: Optional function to extract reference
-            strategy_resolver: Optional function to resolve strategy
+            reference_selector: Optional function to extract reference from record.
+                If provided, this takes precedence over item.reference from strategies.
+            strategy_resolver: Optional function to resolve evaluation strategy.
+                If using a custom reference_selector with DefaultEvaluationStrategy,
+                the selector will take precedence.
+
+        Note:
+            When using DefaultEvaluationStrategy with a custom reference_selector,
+            the reference_selector will override the default behavior. Consider
+            using a custom strategy if you need more control over reference selection.
         """
         self._extractor = extractor
         self._metrics = list(metrics)
-        self._reference_selector = reference_selector or _default_reference_selector
+        self._reference_selector = reference_selector
+        self._has_custom_reference_selector = reference_selector is not None
         self._strategy_resolver = strategy_resolver or (
             lambda record: evaluation_strategies.DefaultEvaluationStrategy()
         )
@@ -102,6 +142,17 @@ class EvaluationPipeline:
             tuple[str, Callable[[core_entities.GenerationRecord], bool]]
         ] = []
 
+        # Validation: warn if custom reference_selector is used with default strategy
+        if self._has_custom_reference_selector and strategy_resolver is None:
+            warnings.warn(
+                "Custom reference_selector provided without custom strategy_resolver. "
+                "The reference_selector will take precedence over DefaultEvaluationStrategy's "
+                "reference handling. If you need more control, consider providing a custom "
+                "strategy_resolver that sets reference=None in EvaluationItem.",
+                UserWarning,
+                stacklevel=2,
+            )
+
     def evaluate(
         self, records: Sequence[core_entities.GenerationRecord]
     ) -> EvaluationReport:
@@ -167,7 +218,16 @@ class EvaluationPipeline:
                             record_failures.append(message)
                             continue
 
-                        reference = item.reference or self._reference_selector(record)
+                        # CRITICAL: Always call reference_selector if provided (takes precedence)
+                        # This fixes the issue where DefaultEvaluationStrategy's reference
+                        # would prevent custom reference_selector from being called
+                        if self._has_custom_reference_selector:
+                            reference = self._reference_selector(record)
+                        elif item.reference is not None:
+                            reference = item.reference
+                        else:
+                            reference = _default_reference_selector(record)
+
                         references = (
                             _normalize_references(reference)
                             if reference is not None

themis/experiment/cache_manager.py

@@ -65,18 +65,21 @@ class CacheManager:
             return {}
         return self._storage.load_cached_records(run_id)
 
-    def load_cached_evaluations(self, run_id: str) -> dict[str, EvaluationRecord]:
+    def load_cached_evaluations(
+        self, run_id: str, evaluation_config: dict | None = None
+    ) -> dict[str, EvaluationRecord]:
         """Load cached evaluation records for resuming.
 
         Args:
             run_id: Unique run identifier
+            evaluation_config: Evaluation configuration (metrics, extractor) for cache matching
 
         Returns:
             Dictionary mapping cache keys to evaluation records
         """
         if not self._enable_resume or self._storage is None:
             return {}
-        return self._storage.load_cached_evaluations(run_id)
+        return self._storage.load_cached_evaluations(run_id, evaluation_config=evaluation_config)
 
     def save_generation_record(
         self,
@@ -99,6 +102,7 @@ class CacheManager:
         run_id: str,
         generation_record: GenerationRecord,
         evaluation_record: EvaluationRecord,
+        evaluation_config: dict | None = None,
     ) -> None:
         """Save a single evaluation record.
 
@@ -106,10 +110,11 @@ class CacheManager:
             run_id: Unique run identifier
             generation_record: Corresponding generation record
             evaluation_record: Evaluation record to save
+            evaluation_config: Evaluation configuration for cache invalidation
         """
         if self._storage is not None and self._enable_cache:
             self._storage.append_evaluation(
-                run_id, generation_record, evaluation_record
+                run_id, generation_record, evaluation_record, evaluation_config=evaluation_config
             )
 
     def get_run_path(self, run_id: str) -> str | None: