evalscope 1.0.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

evalscope/metrics/llm_judge.py

@@ -2,6 +2,7 @@ import os
 import re
 from typing import Any, Dict, List, Optional
 
+from evalscope.api.messages import ChatMessage, ChatMessageSystem, ChatMessageUser
 from evalscope.constants import JudgeScoreType
 from evalscope.utils.logger import get_logger
 
@@ -109,20 +110,31 @@ class LLMJudge:
             config=GenerateConfig(**self.generation_config),
         )
 
-    def judge(self, prompt: str, system_prompt: Optional[str] = None) -> str:
+    def judge(
+        self,
+        prompt: str = '',
+        system_prompt: Optional[str] = None,
+        messages: Optional[List[ChatMessage]] = None
+    ) -> str:
         """
+        Generate a response from the LLM based on the provided prompt and context.
+        If messages is provided, it will be used as the input context.
+
         Args:
             prompt (str): The prompt to evaluate
             system_prompt (str, optional): The system prompt to use for the evaluation
+            messages (List[ChatMessage], optional): A list of chat messages to include in the evaluation
         Returns:
             str: The response from the LLM
         """
-        from evalscope.api.messages import ChatMessageSystem, ChatMessageUser
-
-        system_content = system_prompt or self.system_prompt
-        input_messages = [ChatMessageUser(content=prompt)]
-        if system_content:
-            input_messages.insert(0, ChatMessageSystem(content=system_content))
+        # parse messages
+        if messages is not None:
+            input_messages = messages
+        else:
+            system_content = system_prompt or self.system_prompt
+            input_messages = [ChatMessageUser(content=prompt)]
+            if system_content:
+                input_messages.insert(0, ChatMessageSystem(content=system_content))
         try:
             # Send request using ServerModelAdapter
             response = self.model.generate(input_messages)

evalscope/metrics/math_parser.py

@@ -211,6 +211,11 @@ def strip_answer_string(string):
     # Remove grade level (e.g., 12th grade) and just maintain the integer
     string = re.sub(r'thgrade$', '', string)
 
+    # Normalize thousands-formatted numbers (e.g., 70,000 or -1,234,567.89) by removing commas
+    # This must run before the "list of integers" sorting to avoid misclassifying numbers with thousand separators.
+    if re.fullmatch(r'\s*-?\d{1,3}(?:,\d{3})+(?:\.\d+)?\s*', string):
+        string = string.replace(',', '')
+
     # If the answer is a list of integers (without parenthesis), sort them
     if re.fullmatch(r'(\s*-?\d+\s*,)*\s*-?\d+\s*', string):
         # Split the string into a list of integers
@@ -262,6 +267,8 @@ def extract_answer(pred_str, use_last_number=True):
     elif '答案是' in pred_str:
         # Handle Chinese few-shot multiple choice problem answer extraction
         pred = pred_str.split('答案是')[1].strip().split('\n\n')[0].strip()
+    elif 'ANSWER:' in pred_str:
+        pred = pred_str.split('ANSWER:')[-1].strip()
     else:  # use the last number
         if use_last_number:
             pattern = '-?\d*\.?\d+'
@@ -529,3 +536,10 @@ def symbolic_equal(a, b):
         pass
 
     return False
+
+
+if __name__ == '__main__':
+    print(math_equal('\n\\boxed{70,\\!000}\n', '70000'))
+    print(extract_answer('The answer is \\boxed{70,\\!000}'))
+    print(strip_answer_string(extract_answer('The answer is \\boxed{70,\\!000}')))
+    print(math_equal(extract_answer('The answer is \\boxed{70,\\!000}'), '70000'))

evalscope/metrics/metric.py

@@ -1,16 +1,27 @@
+import json
+import numpy as np
+import os
 from collections import defaultdict
-from typing import List
+from typing import Dict, List
 
-from evalscope.api.metric import Aggregator, AggScore, Metric, SampleScore, T2IMetric
+from evalscope.api.metric import Aggregator, AggScore, Metric, SampleScore, SingletonMetric, T2IMetric
 from evalscope.api.registry import register_aggregation, register_metric
-from .metrics import mean
+from evalscope.utils.import_utils import check_import
+from .metrics import calculate_pass_at_k, calculate_pass_hat_k, mean, normalize_text
+
+# ##################
+# NLP Metrics ######
+# ##################
 
 
 @register_metric(name='exact_match')
 class ExactMatch(Metric):
 
     def apply(self, predictions, references):
-        return [float(prediction == reference) for prediction, reference in zip(predictions, references)]
+        return [
+            float(normalize_text(prediction) == normalize_text(reference))
+            for prediction, reference in zip(predictions, references)
+        ]
 
 
 @register_metric(name='acc')
@@ -30,13 +41,12 @@ class Accuracy(ExactMatch):
                     results.append(0.0)
             return results
         elif self.numeric:
-            from .math_parser import extract_answer, math_equal, strip_answer_string
+            from .math_parser import math_equal, strip_answer_string
 
             results = []
             for prediction, reference in zip(predictions, references):
-                pred_answer = strip_answer_string(extract_answer(prediction))
                 ref_answer = strip_answer_string(reference)
-                results.append(float(math_equal(pred_answer, ref_answer)))
+                results.append(float(math_equal(prediction, ref_answer)))
 
             return results
         else:
@@ -92,9 +102,114 @@ class MultiChoiceAcc(Metric):
         return res
 
 
+@register_metric(name='anls')
+class ANLS(Metric):
+
+    def __init__(self, thresh_hold=0.5):
+        self.thresh_hold = thresh_hold
+
+    def apply(self, predictions, references):
+        """
+        Calculate ANLS (Average Normalized Levenshtein Similarity) for a list of predictions and references.
+        This implementation is adapted from
+        https://github.com/QwenLM/Qwen-VL/blob/master/eval_mm/infographicsvqa_eval.py
+
+        Args:
+            references (List[str]): List of correct answers. Each answer can be a string of json.
+            predictions (List[str]): List of predicted answers.
+        """
+        from .metrics import levenshtein_distance
+
+        res = []
+        # Unwrap predictions if it's a nested list
+        for prediction, reference in zip(predictions, references):
+            # Parse the reference which is a json string
+            try:
+                answer = json.loads(reference)
+            except json.JSONDecodeError:
+                answer = reference
+            if isinstance(answer, str):
+                answer = [answer]
+            assert isinstance(answer, list), 'The reference answer should be a list of answers.'
+
+            # Calculate ANLS for each reference answer
+            values = []
+            for ans in answer:
+                # preprocess both the answers - gt and prediction
+                gt_answer = ' '.join(ans.strip().lower().split())
+                det_answer = ' '.join(prediction.strip().lower().split())
+
+                dist = levenshtein_distance(gt_answer, det_answer)
+                length = max(len(ans.upper()), len(prediction.upper()))
+                values.append(0.0 if length == 0 else float(dist) / float(length))
+
+            question_result = 0.0
+            if values:
+                question_result = 1 - min(values)
+                if question_result < self.thresh_hold:
+                    question_result = 0.0
+            res.append(question_result)
+        return res
+
+
+@register_metric(name='bertscore')
+class BertScore(SingletonMetric):
+
+    def _init_once(self, model_id_or_path: str = 'google-bert/bert-base-chinese', **kwargs):
+        """BertScore metric.
+
+        Args:
+            model_id_or_path (str, optional): The model ID on modelscope or path to the pre-trained model.
+                Defaults to 'google-bert/bert-base-chinese'.
+        """
+        check_import('torch', 'torch', raise_error=True, feature_name='BertScore Metric')
+
+        from .bert_score.scorer import BERTScorer
+        self.scorer = BERTScorer(model_id_or_path=model_id_or_path, batch_size=1024, **kwargs)
+
+    def apply(self, predictions: List[str], references: List[str]) -> List[float]:
+        _, _, F1 = self.scorer.score(predictions, references)
+        return [round(f1.item(), 6) for f1 in F1]
+
+
+@register_metric(name='comet')
+class COMETScore(SingletonMetric):
+
+    def _init_once(self, model_id_or_path: str = 'evalscope/wmt22-comet-da'):
+        """COMETScore metric.
+
+        Args:
+            model_name (str, optional): The model name on huggingface.
+                Defaults to 'evalscope/wmt22-comet-da'.
+        """
+        check_import('comet', 'unbabel-comet', raise_error=True, feature_name='COMETScore Metric')
+
+        from comet import load_from_checkpoint
+        from modelscope import snapshot_download
+
+        self.model_name = model_id_or_path
+        model_path = snapshot_download(model_id_or_path)
+        checkpoint_path = os.path.join(model_path, 'checkpoints', 'model.ckpt')
+        self.comet_scorer = load_from_checkpoint(checkpoint_path)
+
+    def apply(self, samples: List[Dict[str, str]]) -> List[float]:
+        """Apply COMET scoring."""
+        import torch
+
+        model_output = self.comet_scorer.predict(
+            samples=samples,
+            batch_size=1024,
+            gpus=1 if torch.cuda.is_available() else 0,
+            progress_bar=False,
+        )
+        scores = model_output.scores if hasattr(model_output, 'scores') else [model_output.system_score] * len(samples)
+
+        return [round(score, 6) for score in scores]
+
+
 # ##################
 # T2I Metrics ######
-####################
+# ##################
 @register_metric(name='VQAScore')
 class VQAScore(T2IMetric):
 
@@ -202,6 +317,9 @@ class Mean(Aggregator):
 
     name = 'mean'
 
+    def agg_func(self, values: List[float]) -> float:
+        return mean(values)
+
     def __call__(self, scores: List[SampleScore]) -> List[AggScore]:
         """Aggregate scores by computing the mean for each metric.
 
@@ -230,7 +348,7 @@ class Mean(Aggregator):
             if values:  # Only process non-empty value lists
                 aggregated_scores.append(
                     AggScore(
-                        score=mean(values),
+                        score=self.agg_func(values),
                         metric_name=metric_name,
                         aggregation_name=self.name,
                         num=len(values),
@@ -241,6 +359,20 @@ class Mean(Aggregator):
         return aggregated_scores
 
 
+@register_aggregation(name='clipped_mean')
+class ClippedMean(Mean):
+
+    name = 'clipped_mean'
+
+    def __init__(self, clip_min: float = 0.0, clip_max: float = 1.0):
+        self.clip_min = clip_min
+        self.clip_max = clip_max
+
+    def agg_func(self, values: List[float]) -> float:
+        clipped_values = min(max(mean(values), self.clip_min), self.clip_max)
+        return clipped_values
+
+
 @register_aggregation(name='pass_at_k')
 class PassAtK(Aggregator):
 
@@ -260,10 +392,6 @@ class PassAtK(Aggregator):
         if not scores:
             return []
 
-        import numpy as np
-
-        from .metrics import calculate_pass_at_k
-
         # Group scores by metric name and group_id
         metric_groups = defaultdict(lambda: defaultdict(list))
 
@@ -305,3 +433,179 @@ class PassAtK(Aggregator):
                 )
 
         return aggregated_scores
+
+
+@register_aggregation(name='mean_and_pass_at_k')
+class MeanPassAtK(Aggregator):
+
+    def __init__(self):
+        self.name = 'mean_and_pass_at_k'
+
+    def __call__(self, scores: List[SampleScore]) -> List[AggScore]:
+        """Add per-metric pass@k (computed via calculate_pass_at_k) to each sample, then mean-aggregate.
+
+        For each metric:
+        - Group scores by group_id
+        - Collect binary correctness values
+        - Infer k as (total samples / number of groups) assuming uniform repetitions
+        - Compute per-group pass@k via calculate_pass_at_k
+        - Annotate each sample with metric_pass@k for its group
+        Finally run Mean() over the augmented metric set.
+        """
+        if not scores:
+            return []
+
+        # Extract metric names present in score values
+        metrics = list(scores[0].score.value.keys())
+
+        for metric_name in metrics:
+            # group_id -> list[float] (0/1 correctness values)
+            group_values: Dict[str, List[float]] = defaultdict(list)
+            for s in scores:
+                group_id = getattr(s, 'group_id', s.sample_id)
+                value = float(s.score.value[metric_name])
+                group_values[group_id].append(value)
+
+            if not group_values:
+                continue
+
+            # Infer k (assumes roughly uniform repeats)
+            k = int(len(scores) / len(group_values)) if len(group_values) > 0 else 1
+            if k <= 0:
+                k = 1
+
+            # Prepare inputs for calculate_pass_at_k
+            num_samples: List[int] = []
+            num_correct: List[int] = []
+            group_order: List[str] = []
+            for gid, vals in group_values.items():
+                group_order.append(gid)
+                num_samples.append(len(vals))
+                num_correct.append(int(sum(vals)))
+
+            # Compute per-group pass@k
+            pass_at_k_list = calculate_pass_at_k(num_samples, num_correct, k)
+            # Map back: group_id -> pass@k value
+            pass_at_k_map = {gid: float(v) for gid, v in zip(group_order, pass_at_k_list)}
+
+            # Annotate each sample with its group's pass@k
+            for s in scores:
+                group_id = getattr(s, 'group_id', s.sample_id)
+                s.score.value[f'{metric_name}_pass@{k}'] = pass_at_k_map[group_id]
+
+        # Delegate mean aggregation over original + injected pass@k metrics
+        m = Mean()
+        return m(scores)
+
+
+@register_aggregation(name='mean_and_vote_at_k')
+class MeanVoteAtK(Aggregator):
+
+    def __init__(self):
+
+        self.name = 'mean_and_vote_at_k'
+
+    def __call__(self, scores: List[SampleScore]) -> List[AggScore]:
+        """Aggregate scores by computing the vote@k for each metric using group_id.
+
+        Args:
+            scores: List of sample scores to aggregate
+
+        Returns:
+            List of aggregated scores with vote@k values
+        """
+        if not scores:
+            return []
+
+        metrics = list(scores[0].score.value.keys())
+
+        # Calculate vote@k for all metrics
+        for metric_name in metrics:
+
+            # Count of occurrences for each answer in each group_id
+            answer_groups = defaultdict(lambda: defaultdict(int))
+            # Score for each answer in each group_id
+            scores_groups = defaultdict(lambda: defaultdict(float))
+            # Score of the most frequently occurring answer
+            final_scores_groups = defaultdict(float)
+            # Count different answers for this metric
+            for score in scores:
+                group_id = getattr(score, 'group_id', score.sample_id)  # fallback to sample_id if no group_id
+                answer_prediction = getattr(score.score, 'extracted_prediction', None)
+                answer_groups[group_id][answer_prediction] += 1
+                scores_groups[group_id][answer_prediction] = score.score.value[metric_name]
+            # Calculate the repetition count k for each problem
+            k = int(len(scores) / len(answer_groups))
+
+            # Use the score of the most frequently occurring answer as the group's score
+            for group_id in answer_groups:
+                final_scores_groups[group_id] = scores_groups[group_id][
+                    max(answer_groups[group_id], key=answer_groups[group_id].get)]
+
+            # Add the corresponding vote@k for the metric to each score's value
+            for score in scores:
+                group_id = getattr(score, 'group_id', score.sample_id)
+                score.score.value.update({f'{metric_name}_vote@{k}': final_scores_groups[group_id]})
+
+        # Calculate the mean value for all metrics and their corresponding vote@k
+        m = Mean()
+        return m(scores)
+
+
+@register_aggregation(name='mean_and_pass_hat_k')
+class MeanPassHatK(Aggregator):
+
+    def __init__(self):
+        self.name = 'mean_and_pass_hat_k'
+
+    def __call__(self, scores: List[SampleScore]) -> List[AggScore]:
+        """Add per-metric pass^k using calculate_pass_hat_k, then mean-aggregate.
+
+        For each metric:
+        - Group scores by group_id
+        - Collect binary correctness values
+        - Infer k as approximate repeats and clamp to min attempts across groups
+        - Compute per-group pass^k via calculate_pass_hat_k
+        - Annotate each sample with metric_pass^{k} for its group
+        Finally run Mean() over the augmented metric set.
+        """
+        if not scores:
+            return []
+
+        # Freeze metric names before augmenting values to avoid iterating injected keys
+        metrics = list(scores[0].score.value.keys())
+
+        for metric_name in metrics:
+            # group_id -> list[float] (0/1 correctness values)
+            group_values: Dict[str, List[float]] = defaultdict(list)
+            for s in scores:
+                group_id = getattr(s, 'group_id', s.sample_id)
+                value = float(s.score.value[metric_name])
+                group_values[group_id].append(value)
+
+            if not group_values:
+                continue
+
+            # Infer repeats and clamp to the smallest group size to satisfy k <= n
+            approx_k = int(len(scores) / len(group_values)) if len(group_values) > 0 else 1
+            min_n = min(len(vals) for vals in group_values.values())
+            k = max(1, min(approx_k, min_n))
+
+            # Compute per-group pass^k
+            pass_hat_k_map: Dict[str, float] = {}
+            for gid, vals in group_values.items():
+                n = len(vals)
+                c = int(sum(vals))
+                # calculate_pass_hat_k requires k <= n; ensured by clamping above
+                pass_hat_k_map[gid] = float(calculate_pass_hat_k(n, c, k))
+
+            # Annotate each sample with its group's pass^k
+            suffix = f'pass^{k}'
+            injected_key = f'{metric_name}_{suffix}'
+            for s in scores:
+                group_id = getattr(s, 'group_id', s.sample_id)
+                s.score.value[injected_key] = pass_hat_k_map[group_id]
+
+        # Mean aggregate over original + injected pass^k metrics
+        m = Mean()
+        return m(scores)

evalscope/metrics/metrics.py

@@ -12,6 +12,11 @@ from collections.abc import Iterable
 from typing import Dict, List, Union
 
 
+def normalize_text(text: str) -> str:
+    """Normalize text by lowering case and stripping whitespace."""
+    return text.strip().lower()
+
+
 def mean(arr: list):
     if not arr:
         return 0.0
@@ -467,3 +472,35 @@ def calculate_pass_at_k(
         num_samples_it = iter(num_samples)
 
     return np.array([estimator(int(n), int(c), k) for n, c in zip(num_samples_it, num_correct)])
+
+
+def calculate_pass_hat_k(num_trials: int, success_count: int, k: int) -> float:
+    """
+    Compute the pass^k metric for the given number of trials, success count, and k.
+    from https://arxiv.org/pdf/2406.12045
+    Args:
+        num_trials: The number of trials.
+        success_count: The number of successful trials.
+        k: The number of trials to consider.
+    Returns:
+        The pass^k metric.
+    """
+    if num_trials < k:
+        raise ValueError(f'Number of trials {num_trials} is less than k {k}.')
+    return math.comb(success_count, k) / math.comb(num_trials, k)
+
+
+def levenshtein_distance(s1, s2):
+    if len(s1) > len(s2):
+        s1, s2 = s2, s1
+
+    distances = range(len(s1) + 1)
+    for i2, c2 in enumerate(s2):
+        distances_ = [i2 + 1]
+        for i1, c1 in enumerate(s1):
+            if c1 == c2:
+                distances_.append(distances[i1])
+            else:
+                distances_.append(1 + min((distances[i1], distances[i1 + 1], distances_[-1])))
+        distances = distances_
+    return distances[-1]

evalscope/metrics/t2v_metrics/models/vqascore_models/lavis/models/blip2_models/Qformer.py

@@ -30,13 +30,9 @@ from transformers.modeling_outputs import (
     SequenceClassifierOutput,
     TokenClassifierOutput,
 )
-from transformers.modeling_utils import (
-    PreTrainedModel,
-    apply_chunking_to_forward,
-    find_pruneable_heads_and_indices,
-    prune_linear_layer,
-)
+from transformers.modeling_utils import PreTrainedModel
 from transformers.models.bert.configuration_bert import BertConfig
+from transformers.pytorch_utils import apply_chunking_to_forward, find_pruneable_heads_and_indices, prune_linear_layer
 from transformers.utils import logging
 from typing import Any, Dict, Optional, Tuple
 

evalscope/metrics/t2v_metrics/models/vqascore_models/lavis/models/blip_models/nlvr_encoder.py

@@ -14,13 +14,9 @@ from transformers.modeling_outputs import (
     BaseModelOutputWithPastAndCrossAttentions,
     BaseModelOutputWithPoolingAndCrossAttentions,
 )
-from transformers.modeling_utils import (
-    PreTrainedModel,
-    apply_chunking_to_forward,
-    find_pruneable_heads_and_indices,
-    prune_linear_layer,
-)
+from transformers.modeling_utils import PreTrainedModel
 from transformers.models.bert.configuration_bert import BertConfig
+from transformers.pytorch_utils import apply_chunking_to_forward, find_pruneable_heads_and_indices, prune_linear_layer
 from transformers.utils import logging
 from typing import Tuple
 

evalscope/metrics/t2v_metrics/models/vqascore_models/lavis/models/med.py

@@ -31,13 +31,9 @@ from transformers.modeling_outputs import (
     SequenceClassifierOutput,
     TokenClassifierOutput,
 )
-from transformers.modeling_utils import (
-    PreTrainedModel,
-    apply_chunking_to_forward,
-    find_pruneable_heads_and_indices,
-    prune_linear_layer,
-)
+from transformers.modeling_utils import PreTrainedModel
 from transformers.models.bert.configuration_bert import BertConfig
+from transformers.pytorch_utils import apply_chunking_to_forward, find_pruneable_heads_and_indices, prune_linear_layer
 from transformers.utils import logging
 from typing import Optional, Tuple