evalscope 0.10.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

evalscope/metrics/metrics.py

@@ -9,13 +9,48 @@ import random
 import sacrebleu
 from collections import defaultdict
 from collections.abc import Iterable
-from typing import TYPE_CHECKING, Dict, List, Union
+from typing import Dict, List, Union
 
 
-def mean(arr):
+def normalize_text(text: str) -> str:
+    """Normalize text by lowering case and stripping whitespace."""
+    return text.strip().lower()
+
+
+def mean(arr: list):
+    if not arr:
+        return 0.0
+
+    if isinstance(arr[0], list):
+        arr = [item for sublist in arr for item in sublist]
     return sum(arr) / len(arr)
 
 
+def pass_at_k(arr: Union[List[int], List[List[int]]], k: int = 1) -> float:
+    """
+    Calculates the pass@k metric using the calculate_pass_at_k function.
+
+    Args:
+        arr: List of binary values (1 for correct, 0 for incorrect) or list of such lists
+        k: Number of attempts allowed
+
+    Returns:
+        The average pass@k score across all problems
+    """
+    if not arr:
+        return 0.0
+    if not isinstance(arr[0], list):
+        # If arr is a simple list of binary results, convert it to a list of lists
+        arr = [arr]
+
+    # For list of lists case, each inner list represents attempts for one problem
+    num_samples = [len(sub_arr) for sub_arr in arr]
+    num_correct = [sum(sub_arr) for sub_arr in arr]
+    pass_at_k_values = calculate_pass_at_k(num_samples, num_correct, k)
+
+    return float(np.mean(pass_at_k_values))
+
+
 def pop_stddev(arr):
     mu = mean(arr)
     return math.sqrt(sum([(x - mu)**2 for x in arr]) / len(arr))
@@ -43,6 +78,18 @@ def matthews_corrcoef(items):
     return sklearn.metrics.matthews_corrcoef(golds, preds)
 
 
+def simple_f1_score(scores: tuple) -> float:
+    score1 = scores[0]
+    score2 = scores[1]
+    score1 = np.mean(score1) if len(score1) > 0 else 0.0
+    score2 = np.mean(score2) if len(score2) > 0 else 0.0
+
+    if score1 == 0 and score2 == 0:
+        return 0.0
+    else:
+        return 2 * score1 * score2 / (score1 + score2)
+
+
 def f1_score(items):
     import sklearn.metrics
 
@@ -111,11 +158,17 @@ def weighted_mean(items: List) -> float:
 
 
 def micro_mean(items):
-    return sum([item.score * item.num for item in items]) / sum([item.num for item in items])
+    try:
+        return sum([item.score * item.num for item in items]) / sum([item.num for item in items])
+    except ZeroDivisionError:
+        return 0.0
 
 
 def macro_mean(items):
-    return sum([item.score for item in items]) / len(items)
+    try:
+        return sum([item.score for item in items]) / len(items)
+    except ZeroDivisionError:
+        return 0.0
 
 
 def weighted_perplexity(items):
@@ -143,7 +196,7 @@ def bleu(items):
     return sacrebleu.corpus_bleu(preds, refs).score
 
 
-def bleu_ngram_one_sample(predict, reference):
+def bleu_ngram_one_sample(predict: str, reference: str):
     """
     Calculate BLEU-1, BLEU-2, BLEU-3, and BLEU-4 scores
 
@@ -187,7 +240,7 @@ def chrf(items):
     Source: https://github.com/m-popovic/chrF
     Paper: https://www.aclweb.org/anthology/W15-3049.pdf
 
-    Higher is better  # TODO I think
+    Higher is better
     """
     refs = list(zip(*items))[0]
     preds = list(zip(*items))[1]
@@ -274,11 +327,11 @@ def bootstrap_stderr(f, xs, iters):
 
     print('bootstrapping for stddev:', f.__name__)
     for bootstrap in tqdm(
-            pool.imap(
-                _bootstrap_internal(f, chunk_size),
-                [(i, xs) for i in range(iters // chunk_size)],
-            ),
-            total=iters // chunk_size,
+        pool.imap(
+            _bootstrap_internal(f, chunk_size),
+            [(i, xs) for i in range(iters // chunk_size)],
+        ),
+        total=iters // chunk_size,
     ):
         # sample w replacement
         res.extend(bootstrap)
@@ -313,15 +366,17 @@ def yesno(x):
         return 'no'
 
 
-def compute_elo(battles,
-                col_model_a='model_a',
-                col_model_b='model_b',
-                col_win='win',
-                tie_values=['tie', 'tie (bothbad)'],
-                k=32,
-                scale=400,
-                base=10,
-                init_rating=1000):
+def compute_elo(
+    battles,
+    col_model_a='model_a',
+    col_model_b='model_b',
+    col_win='win',
+    tie_values=['tie', 'tie (bothbad)'],
+    k=32,
+    scale=400,
+    base=10,
+    init_rating=1000
+):
     rating = defaultdict(lambda: init_rating)
 
     for rd, model_a, model_b, win in battles[[col_model_a, col_model_b, col_win]].itertuples():
@@ -386,9 +441,11 @@ def calculate_arc_accuracy(question_answers: Dict[str, str], predictions: Dict[s
     return score / len(question_answers)
 
 
-def calculate_pass_at_k(num_samples: Union[int, List[int], np.ndarray],
-                        num_correct: Union[List[int], np.ndarray],
-                        k: int = 1) -> np.ndarray:
+def calculate_pass_at_k(
+    num_samples: Union[int, List[int], np.ndarray],
+    num_correct: Union[List[int], np.ndarray],
+    k: int = 1
+) -> np.ndarray:
     """
     Estimates pass@k of each problem and returns them in an array.
     Examples:
@@ -415,3 +472,35 @@ def calculate_pass_at_k(num_samples: Union[int, List[int], np.ndarray],
         num_samples_it = iter(num_samples)
 
     return np.array([estimator(int(n), int(c), k) for n, c in zip(num_samples_it, num_correct)])
+
+
+def calculate_pass_hat_k(num_trials: int, success_count: int, k: int) -> float:
+    """
+    Compute the pass^k metric for the given number of trials, success count, and k.
+    from https://arxiv.org/pdf/2406.12045
+    Args:
+        num_trials: The number of trials.
+        success_count: The number of successful trials.
+        k: The number of trials to consider.
+    Returns:
+        The pass^k metric.
+    """
+    if num_trials < k:
+        raise ValueError(f'Number of trials {num_trials} is less than k {k}.')
+    return math.comb(success_count, k) / math.comb(num_trials, k)
+
+
+def levenshtein_distance(s1, s2):
+    if len(s1) > len(s2):
+        s1, s2 = s2, s1
+
+    distances = range(len(s1) + 1)
+    for i2, c2 in enumerate(s2):
+        distances_ = [i2 + 1]
+        for i1, c1 in enumerate(s1):
+            if c1 == c2:
+                distances_.append(distances[i1])
+            else:
+                distances_.append(1 + min((distances[i1], distances[i1 + 1], distances_[-1])))
+        distances = distances_
+    return distances[-1]

evalscope/metrics/rouge_metric.py

@@ -19,10 +19,6 @@ class DummyTokenizer:
         return text.split()
 
 
-scorer = rouge_scorer.RougeScorer(['rouge1', 'rouge2', 'rougeL'], tokenizer=DummyTokenizer())
-zh_scorer = Rouge()
-
-
 def is_contains_chinese(strs):
     for _char in strs:
         if '\u4e00' <= _char <= '\u9fa5':
@@ -51,6 +47,7 @@ def compute_rouge_score(predict_l, reference_l):
 
 def compute_rouge_score_one_sample_zh(predict, reference):
     result = dict()
+    zh_scorer = Rouge()
     for p, r in zip(predict, reference):
         p = ' '.join(jieba.cut(p)) if is_contains_chinese(p) else p
         r = ' '.join(jieba.cut(r)) if is_contains_chinese(r) else r
@@ -60,21 +57,22 @@ def compute_rouge_score_one_sample_zh(predict, reference):
         except Exception as e:
             logger.warning(f'rouge score error: {p} {r} {e}')
             continue
-        result['rouge-1-r'] = score['rouge-1']['r']
-        result['rouge-1-p'] = score['rouge-1']['p']
-        result['rouge-1-f'] = score['rouge-1']['f']
-        result['rouge-2-r'] = score['rouge-2']['r']
-        result['rouge-2-p'] = score['rouge-2']['p']
-        result['rouge-2-f'] = score['rouge-2']['f']
-        result['rouge-l-r'] = score['rouge-l']['r']
-        result['rouge-l-p'] = score['rouge-l']['p']
-        result['rouge-l-f'] = score['rouge-l']['f']
+        result['Rouge-1-R'] = score['rouge-1']['r']
+        result['Rouge-1-P'] = score['rouge-1']['p']
+        result['Rouge-1-F'] = score['rouge-1']['f']
+        result['Rouge-2-R'] = score['rouge-2']['r']
+        result['Rouge-2-P'] = score['rouge-2']['p']
+        result['Rouge-2-F'] = score['rouge-2']['f']
+        result['Rouge-L-R'] = score['rouge-l']['r']
+        result['Rouge-L-P'] = score['rouge-l']['p']
+        result['Rouge-L-F'] = score['rouge-l']['f']
 
     return result
 
 
 def compute_rouge_score_one_sample(predict, reference):
     result = dict()
+    scorer = rouge_scorer.RougeScorer(['rouge1', 'rouge2', 'rougeL'], tokenizer=DummyTokenizer())
     for p, r in zip(predict, reference):
         try:
             score = scorer.score(p, r)

evalscope/metrics/t2v_metrics/clipscore.py

@@ -0,0 +1,14 @@
+from typing import List
+
+from .constants import CACHE_DIR
+from .models.clipscore_models import get_clipscore_model, list_all_clipscore_models
+from .score import Score
+
+
+class CLIPScore(Score):
+
+    def prepare_scoremodel(self, model='openai:ViT-L/14', device='cuda', cache_dir=CACHE_DIR):
+        return get_clipscore_model(model, device=device, cache_dir=cache_dir)
+
+    def list_all_models(self) -> List[str]:
+        return list_all_clipscore_models()

evalscope/metrics/t2v_metrics/constants.py

@@ -0,0 +1,12 @@
+import os
+from modelscope.utils.file_utils import get_model_cache_root
+
+CACHE_DIR = get_model_cache_root()
+os.environ['TORCH_HOME'] = CACHE_DIR  # set timm cache dir
+
+# For CLIP-FlanT5
+CONTEXT_LEN = 2048
+SYSTEM_MSG = "A chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions."
+IGNORE_INDEX = -100
+IMAGE_TOKEN_INDEX = -200
+DEFAULT_IMAGE_TOKEN = '<image>'

evalscope/metrics/t2v_metrics/itmscore.py

@@ -0,0 +1,14 @@
+from typing import List
+
+from .constants import CACHE_DIR
+from .models.itmscore_models import get_itmscore_model, list_all_itmscore_models
+from .score import Score
+
+
+class ITMScore(Score):
+
+    def prepare_scoremodel(self, model='blip2-itm', device='cuda', cache_dir=CACHE_DIR):
+        return get_itmscore_model(model, device=device, cache_dir=cache_dir)
+
+    def list_all_models(self) -> List[str]:
+        return list_all_itmscore_models()

evalscope/metrics/t2v_metrics/models/clipscore_models/__init__.py

@@ -0,0 +1,30 @@
+from ...constants import CACHE_DIR
+from .clip_model import CLIP_MODELS, CLIPScoreModel
+from .hpsv2_model import HPSV2_MODELS, HPSV2ScoreModel
+from .mps_model import MPS_MODELS, MPSModel
+from .pickscore_model import PICKSCORE_MODELS, PickScoreModel
+
+ALL_CLIP_MODELS = [
+    CLIP_MODELS,
+    HPSV2_MODELS,
+    PICKSCORE_MODELS,
+    MPS_MODELS,
+]
+
+
+def list_all_clipscore_models():
+    return [model for models in ALL_CLIP_MODELS for model in models]
+
+
+def get_clipscore_model(model_name, device='cuda', cache_dir=CACHE_DIR):
+    assert model_name in list_all_clipscore_models()
+    if model_name in CLIP_MODELS:
+        return CLIPScoreModel(model_name, device=device, cache_dir=cache_dir)
+    elif model_name in HPSV2_MODELS:
+        return HPSV2ScoreModel(model_name, device=device, cache_dir=cache_dir)
+    elif model_name in PICKSCORE_MODELS:
+        return PickScoreModel(model_name, device=device, cache_dir=cache_dir)
+    elif model_name in MPS_MODELS:
+        return MPSModel(model_name, device=device, cache_dir=cache_dir)
+    else:
+        raise NotImplementedError()

evalscope/metrics/t2v_metrics/models/clipscore_models/build_mps_model/base_model.py

@@ -0,0 +1,6 @@
+from dataclasses import dataclass
+
+
+@dataclass
+class BaseModelConfig:
+    pass

evalscope/metrics/t2v_metrics/models/clipscore_models/build_mps_model/clip_model.py

@@ -0,0 +1,134 @@
+import torch
+from dataclasses import dataclass
+from torch import einsum, nn
+from transformers import CLIPConfig
+from transformers import CLIPModel as HFCLIPModel
+from typing import Any, Optional, Tuple, Union
+
+from .base_model import BaseModelConfig
+from .cross_modeling import Cross_model
+
+
+class XCLIPModel(HFCLIPModel):
+
+    def __init__(self, config: CLIPConfig):
+        super().__init__(config)
+
+    def get_text_features(
+        self,
+        input_ids: Optional[torch.Tensor] = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.Tensor] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        return_dict: Optional[bool] = None,
+    ) -> torch.FloatTensor:
+
+        # Use CLIP model's config for some fields (if specified) instead of those of vision & text components.
+        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+        output_hidden_states = (
+            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+        )
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+        text_outputs = self.text_model(
+            input_ids=input_ids,
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+        )
+
+        # pooled_output = text_outputs[1]
+        # text_features = self.text_projection(pooled_output)
+        last_hidden_state = text_outputs[0]
+        text_features = self.text_projection(last_hidden_state)
+
+        pooled_output = text_outputs[1]
+        text_features_EOS = self.text_projection(pooled_output)
+
+        # del last_hidden_state, text_outputs
+        # gc.collect()
+
+        return text_features, text_features_EOS
+
+    def get_image_features(
+        self,
+        pixel_values: Optional[torch.FloatTensor] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        return_dict: Optional[bool] = None,
+    ) -> torch.FloatTensor:
+
+        # Use CLIP model's config for some fields (if specified) instead of those of vision & text components.
+        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+        output_hidden_states = (
+            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+        )
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+        vision_outputs = self.vision_model(
+            pixel_values=pixel_values,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+        )
+
+        # pooled_output = vision_outputs[1]  # pooled_output
+        # image_features = self.visual_projection(pooled_output)
+        last_hidden_state = vision_outputs[0]
+        image_features = self.visual_projection(last_hidden_state)
+
+        return image_features
+
+
+@dataclass
+class ClipModelConfig(BaseModelConfig):
+    _target_: str = 'trainer.models.clip_model.CLIPModel'
+    pretrained_model_name_or_path: str = 'openai/clip-vit-base-patch32'
+
+
+class CLIPModel(nn.Module):
+
+    def __init__(self, config):
+        super().__init__()
+        self.model = XCLIPModel._from_config(config)
+        self.cross_model = Cross_model(dim=1024, layer_num=4, heads=16)
+
+    def get_text_features(self, *args, **kwargs):
+        return self.model.get_text_features(*args, **kwargs)
+
+    def get_image_features(self, *args, **kwargs):
+        return self.model.get_image_features(*args, **kwargs)
+
+    def forward(self, text_inputs=None, image_inputs=None, condition_inputs=None):
+        outputs = ()
+
+        text_f, text_EOS = self.model.get_text_features(text_inputs)  # B*77*1024
+        outputs += text_EOS,
+
+        image_f = self.model.get_image_features(image_inputs.half())  # 2B*257*1024
+        condition_f, _ = self.model.get_text_features(condition_inputs)  # B*5*1024
+
+        sim_text_condition = einsum('b i d, b j d -> b j i', text_f, condition_f)
+        sim_text_condition = torch.max(sim_text_condition, dim=1, keepdim=True)[0]
+        sim_text_condition = sim_text_condition / sim_text_condition.max()
+        mask = torch.where(sim_text_condition > 0.01, 0, float('-inf'))  # B*1*77
+
+        mask = mask.repeat(1, image_f.shape[1], 1)  # B*257*77
+        bc = int(image_f.shape[0] / 2)
+
+        sim0 = self.cross_model(image_f[:bc, :, :], text_f, mask.half())
+        sim1 = self.cross_model(image_f[bc:, :, :], text_f, mask.half())
+        outputs += sim0[:, 0, :],
+        outputs += sim1[:, 0, :],
+
+        return outputs
+
+    @property
+    def logit_scale(self):
+        return self.model.logit_scale
+
+    def save(self, path):
+        self.model.save_pretrained(path)