evalscope 0.14.0__py3-none-any.whl → 0.15.1__py3-none-any.whl

evalscope/benchmarks/live_code_bench/evaluate_utils.py

@@ -2,7 +2,6 @@ import json
 import multiprocessing
 import numpy as np
 from collections import defaultdict
-from concurrent.futures import ProcessPoolExecutor, as_completed
 
 from evalscope.utils.logger import get_logger
 from .pass_k_utils import compute_metrics_from_results
@@ -31,7 +30,10 @@ def codegen_check_correctness(sample, generation, timeout, debug=True):
         args=(sample, generation, debug, result, metadata_list, timeout),
     )
     p.start()
-    p.join(timeout=(timeout + 1) * len(json.loads(sample['input_output'])['inputs']) + 5)
+    global_timeout = (timeout + 1) * len(json.loads(sample['input_output'])['inputs'])
+    if debug:
+        logger.info(f'global timeout = {global_timeout}')
+    p.join(timeout=global_timeout)
     if p.is_alive():
         p.kill()
     if not result:
@@ -39,7 +41,7 @@ def codegen_check_correctness(sample, generation, timeout, debug=True):
         # consider that all tests failed
         result = [[-1 for i in range(len(in_outs['inputs']))]]
         if debug:
-            logger.info('global timeout')
+            logger.info('global timeout occured: alarm went off')
     return result[0], metadata_list[0]
 
 
@@ -99,7 +101,7 @@ def evaluate_generations(
     samples_list: list,
     generations_list: list[list[str]],
     debug: bool = False,
-    num_process_evaluate: int = 16,
+    num_process_evaluate: int = 16,  # This parameter will be unused
     timeout=6,
 ):
     """We take the list of code generations and try to compile them and the run
@@ -117,26 +119,19 @@ def evaluate_generations(
         [-2] = compile error, [-1] = runtime error [False] = failed test
             case [True] = passed test case
     """
+    results = {}
+    metadata = {}
 
-    # generations are code generations in the same order of the dataset
-
-    inputs = [[(generations_list[index], samples_list[index], debug, timeout), index]
-              for index in range(len(generations_list))]
-
-    with ProcessPoolExecutor(max_workers=1 if debug else num_process_evaluate) as executor:
-        futures = {
-            executor.submit(evaluate_generations_by_problem, problem_generations, sample, debug, timeout): index
-            for (problem_generations, sample, debug, timeout), index in inputs
-        }
+    for index in range(len(generations_list)):
+        problem_generations = generations_list[index]
+        sample = samples_list[index]
 
-        results = {}
-        metadata = {}
-        for future in as_completed(futures):
-            index = futures[future]
-            results[index], metadata[index] = future.result()
+        result, meta = evaluate_generations_by_problem(problem_generations, sample, debug, timeout)
+        results[index] = result
+        metadata[index] = meta
 
-    assert len(results) == len(inputs), f'results = {len(results)} inputs = {len(inputs)} {results=}'
-    # results = {i: r for r, (_, i) in zip(results, inputs)}
+    assert len(results) == len(
+        generations_list), f'results = {len(results)} inputs = {len(generations_list)} {results=}'
 
     return results, metadata
 

evalscope/benchmarks/live_code_bench/live_code_bench_adapter.py

@@ -18,7 +18,8 @@ logger = get_logger()
     extra_params={
         'start_date': None,
         'end_date': None,
-        'timeout': 6
+        'timeout': 6,
+        'debug': False
     },
     system_prompt=
     'You are an expert Python programmer. You will be given a question (problem specification) and will generate a correct Python program that matches the specification and passes all tests. You will NOT return anything except for the program.',  # noqa: E501
@@ -33,6 +34,7 @@ class LiveCodeBenchAdapter(DataAdapter):
         extra_params = kwargs.get('extra_params', {})
 
         self.timeout = extra_params.get('timeout', 6)
+        self.debug = extra_params.get('debug', False)
         self.start_date = extra_params.get('start_date')
         self.end_date = extra_params.get('end_date')
 
@@ -84,5 +86,6 @@ class LiveCodeBenchAdapter(DataAdapter):
             k_list=[1],
             num_process_evaluate=1,
             timeout=self.timeout,
+            debug=self.debug,
         )
         return metrics['pass@1'] / 100  # convert to point scale

evalscope/benchmarks/live_code_bench/testing_util.py

@@ -12,6 +12,7 @@ import time
 from datetime import datetime
 from decimal import Decimal
 from enum import Enum
+from functools import partial
 from io import StringIO
 # from pyext import RuntimeModule
 from types import ModuleType
@@ -46,8 +47,9 @@ class TimeoutException(Exception):
     pass
 
 
-def timeout_handler(signum, frame):
-    logger.info('timeout occured: alarm went off')
+def timeout_handler(debug, signum, frame):
+    if debug:
+        logger.info('timeout occured: alarm went off')
     raise TimeoutException
 
 
@@ -381,7 +383,8 @@ def run_test(sample, test=None, debug=False, timeout=6):
     if test(generated_code) is not None it'll try to run the code.
     otherwise it'll just return an input and output pair.
     """
-    signal.signal(signal.SIGALRM, timeout_handler)
+    timeout_handler_wrapper = partial(timeout_handler, debug)
+    signal.signal(signal.SIGALRM, timeout_handler_wrapper)
 
     # Disable functionalities that can make destructive changes to the test.
     # max memory is set to 4GB

evalscope/benchmarks/math_500/math_500_adapter.py

@@ -1,5 +1,5 @@
 from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.metrics.math_parser import extract_answer, math_equal, strip_answer_string
+from evalscope.metrics import extract_answer, math_equal, strip_answer_string
 from evalscope.utils.logger import get_logger
 
 # flake8: noqa

evalscope/benchmarks/mmlu/mmlu_adapter.py

@@ -137,7 +137,7 @@ SUBJECT_MAPPING = {
     name='mmlu',
     pretty_name='MMLU',
     dataset_id='modelscope/mmlu',
-    model_adapter=OutputType.MULTIPLE_CHOICE,
+    model_adapter=OutputType.GENERATION,
     output_types=[OutputType.MULTIPLE_CHOICE, OutputType.GENERATION],
     subset_list=SUBSET_LIST,
     metric_list=['AverageAccuracy'],
@@ -263,6 +263,8 @@ class MMLUAdapter(DataAdapter):
 
         if include_answer:
             example += f"\nAnswer: {input_d['target']}\n\n"
+        else:
+            example += '\nAnswer: \n\n'
 
         return example
 

evalscope/benchmarks/simple_qa/simple_qa_adapter.py

@@ -3,8 +3,7 @@ from collections import defaultdict
 from typing import Any, List
 
 from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.metrics import Metric, mean, metric_registry
-from evalscope.metrics.llm_judge import LLMJudge
+from evalscope.metrics import LLMJudge, Metric, mean, metric_registry
 from evalscope.utils.logger import get_logger
 
 # flake8: noqa

evalscope/benchmarks/utils.py

@@ -1,6 +1,6 @@
-from dataclasses import dataclass
+from dataclasses import asdict, dataclass
 from functools import wraps
-from typing import Dict, List, Optional
+from typing import Dict, List, Optional, Union
 
 from evalscope.constants import EvalType
 from evalscope.utils.filters import Filter
@@ -9,30 +9,21 @@ from evalscope.utils.filters import Filter
 @dataclass
 class PromptData:
     data: List[str]
-    index: Optional[int] = 0
+    index: Optional[Union[int, str]] = 0
     system_prompt: Optional[str] = None
     multi_choices: Optional[List[str]] = None
+    id: Optional[str] = None
 
     def to_dict(self) -> Dict:
-        if self.multi_choices is None:
-            return {
-                'data': self.data,
-                'index': self.index,
-                'system_prompt': self.system_prompt,
-            }
-        else:
-            return {
-                'data': self.data,
-                'index': self.index,
-                'system_prompt': self.system_prompt,
-                'multi_choices': self.multi_choices,
-            }
+        return {k: v for k, v in asdict(self).items() if v is not None}
 
 
 def preprocess_decorator(func):
 
     @wraps(func)
     def wrapper(self, result: str, raw_input_d: dict = None, eval_type: str = EvalType.CHECKPOINT):
+        if result is None:
+            result = ''
         filters = self.config_kwargs.get('filters', None)
         if filters:
             # Apply filters to the resultply filters to the result

evalscope/cli/start_app.py

@@ -21,7 +21,7 @@ class StartAppCMD(CLICommand):
     def define_args(parsers: ArgumentParser):
         """ define args for create pipeline template command.
         """
-        from evalscope.report.app import add_argument
+        from evalscope.report import add_argument
 
         parser = parsers.add_parser(StartAppCMD.name)
         add_argument(parser)

evalscope/collections/evaluator.py

@@ -1,8 +1,10 @@
 import json
 import os
 import pandas as pd
+import random
 from collections import defaultdict
 from concurrent.futures import ThreadPoolExecutor, as_completed
+from copy import deepcopy
 from tabulate import tabulate
 from tqdm import tqdm
 from typing import List
@@ -10,7 +12,7 @@ from typing import List
 from evalscope.benchmarks import Benchmark, DataAdapter
 from evalscope.collections.sampler import DatasetEntry
 from evalscope.config import TaskConfig
-from evalscope.constants import AnswerKeys, DumpMode, EvalType
+from evalscope.constants import AnswerKeys, DataCollection, DumpMode, EvalType
 from evalscope.evaluator import Evaluator
 from evalscope.models import initialize_model_adapter
 from evalscope.report import ReportGenerator
@@ -67,9 +69,10 @@ class EvaluatorCollection:
     def load(self) -> tuple[list[DatasetEntry], str]:
         dataset_name = os.path.splitext(os.path.basename(self.data_adapter.dataset_id))[0]
         raw_dataset = self.data_adapter.load()
-        # limit the dataset
+        # random limit the dataset
         if self.task_cfg.limit:
-            raw_dataset = raw_dataset[:self.task_cfg.limit]
+            raw_dataset = random.sample(raw_dataset,
+                                        self.task_cfg.limit) if len(raw_dataset) > self.task_cfg.limit else raw_dataset
         # index dataset
         datasets = []
         for sample in raw_dataset:
@@ -95,10 +98,17 @@ class EvaluatorCollection:
 
     def _initialize_evaluators(self):
         evaluators = {}
+        # load dataset args
+        dataset_args = deepcopy(self.task_cfg.dataset_args)
+        common_args = dataset_args.get(DataCollection.NAME, {})
         for dataset_name in self.dataset_name_map.keys():
             benchmark = Benchmark.get(dataset_name)
             model_adapter = initialize_model_adapter(self.task_cfg, benchmark, self.model)
-            data_adapter = benchmark.get_data_adapter()
+            # update dataset args
+            cur_dataset_args = dataset_args.get(dataset_name, {})
+            cur_dataset_args.update(common_args)
+            # get data adapter
+            data_adapter = benchmark.get_data_adapter(cur_dataset_args)
             evaluators[dataset_name] = SimpleEvaluator(dataset_name, data_adapter, model_adapter, self.task_cfg,
                                                        self.outputs)
         return evaluators
@@ -185,12 +195,14 @@ class EvaluatorCollection:
                 index = answer.get(AnswerKeys.INDEX)
                 answer_dict[index] = answer
                 indices.add(index)
+
             data = []
             for sample in self.dataset:
                 if sample.index not in indices:
                     data.append(sample)
             data_map = self._init_name_map(data)
 
+            logger.info(f'Reuse from {pred_file_path}. Loaded {len(indices)} samples, remain {len(data)} samples.')
             return answer_dict, data, data_map
         return answer_dict, self.dataset, self.dataset_name_map
 

evalscope/config.py

@@ -4,13 +4,12 @@ import copy
 import json
 import os
 from argparse import Namespace
-from collections import OrderedDict
 from dataclasses import dataclass, field
 from typing import Dict, List, Optional, Union
 
 from evalscope.constants import (DEFAULT_DATASET_CACHE_DIR, DEFAULT_WORK_DIR, EvalBackend, EvalStage, EvalType, HubType,
-                                 JudgeStrategy, OutputType)
-from evalscope.models.custom import CustomModel
+                                 JudgeStrategy, ModelTask, OutputType)
+from evalscope.models import CustomModel, DummyCustomModel
 from evalscope.utils import gen_hash
 from evalscope.utils.io_utils import dict_to_yaml, json_to_dict, yaml_to_dict
 from evalscope.utils.logger import get_logger
@@ -36,6 +35,7 @@ class TaskConfig:
     model: Union[str, 'CustomModel', None] = None
     model_id: Optional[str] = None
     model_args: Optional[Dict] = field(default_factory=lambda: DEFAULT_MODEL_ARGS | {})
+    model_task: Optional[str] = ModelTask.TEXT_GENERATION
 
     # Template-related arguments
     template_type: Optional[str] = None  # Deprecated, will be removed in v1.0.0.
@@ -79,6 +79,10 @@ class TaskConfig:
     judge_model_args: Optional[Dict] = field(default_factory=lambda: {})
 
     def __post_init__(self):
+        if self.model is None:
+            self.model = DummyCustomModel()
+            self.eval_type = EvalType.CUSTOM
+
         if (not self.model_id) and self.model:
             if isinstance(self.model, CustomModel):
                 self.model_id = self.model.config.get('model_id', 'custom_model')

evalscope/constants.py

@@ -1,4 +1,9 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
+# flake8: noqa
+import os
+
+os.environ['MODELSCOPE_LOG_LEVEL'] = '40'  # Set default log level to ERROR
+
 from modelscope.utils.constant import DEFAULT_REPOSITORY_REVISION
 from modelscope.utils.file_utils import get_dataset_cache_root, get_model_cache_root
 
@@ -145,6 +150,7 @@ class OutputType:
     GENERATION = 'generation'  # for text generation tasks and general tasks
     MULTIPLE_CHOICE = 'multiple_choice_logits'  # for multiple choice tasks
     CONTINUOUS = 'continuous_logits'  # for continuous tasks
+    IMAGE_GENERATION = 'image_generation'  # for image generation tasks
 
 
 class EvalBackend:
@@ -164,3 +170,8 @@ class JudgeStrategy:
     RULE = 'rule'
     LLM = 'llm'
     LLM_RECALL = 'llm_recall'
+
+
+class ModelTask:
+    TEXT_GENERATION = 'text_generation'
+    IMAGE_GENERATION = 'image_generation'

evalscope/evaluator/evaluator.py

@@ -66,7 +66,7 @@ class Evaluator(object):
         if self.task_cfg.judge_strategy == JudgeStrategy.RULE:
             self.judge = None
         else:
-            from evalscope.metrics.llm_judge import LLMJudge
+            from evalscope.metrics import LLMJudge
             self.judge = LLMJudge(**self.task_cfg.judge_model_args)
 
     def load_dataset(self):
@@ -281,7 +281,7 @@ class Evaluator(object):
         os.makedirs(os.path.dirname(review_file_path), exist_ok=True)
 
         if self.use_cache and os.path.exists(review_file_path):
-            logger.warning(f'Ignore use_cache={self.use_cache}, updating the review file: {review_file_path} ...')
+            logger.info(f'Updating the review file: {review_file_path} ...')
             os.remove(review_file_path)
 
         def process_single_review(answer_d):
@@ -317,6 +317,8 @@ class Evaluator(object):
         """
 
         review_res_list = []
+        max_choices = max(
+            len(review_d[AnswerKeys.CHOICES]) for review_d in reviews_list if review_d[ReviewKeys.REVIEWED])
         for review_d in reviews_list:
             if not review_d[ReviewKeys.REVIEWED]:
                 logger.warning(f'Review not finished for answer_id: {review_d[AnswerKeys.ANSWER_ID]}')
@@ -325,10 +327,14 @@ class Evaluator(object):
             if len(review_d[AnswerKeys.CHOICES]) == 0:
                 logger.warning(f'No choices found for answer_id: {review_d[AnswerKeys.ANSWER_ID]}')
                 continue
-            elif len(review_d[AnswerKeys.CHOICES]) == 1:
+            elif len(review_d[AnswerKeys.CHOICES]) == 1 and max_choices == 1:
                 review_res = review_d[AnswerKeys.CHOICES][0][ReviewKeys.REVIEW][ReviewKeys.RESULT]
             else:
                 review_res = [choice[ReviewKeys.REVIEW][ReviewKeys.RESULT] for choice in review_d[AnswerKeys.CHOICES]]
+                if len(review_d[AnswerKeys.CHOICES]) < max_choices:
+                    logger.warning(
+                        f'Less choices found for answer_id: {review_d[AnswerKeys.ANSWER_ID]}, '
+                        f'max_choices is {max_choices}, but only {len(review_d[AnswerKeys.CHOICES])} choices found')
 
             review_res_list.append(review_res)
 

evalscope/evaluator/reviewer/auto_reviewer.py

@@ -11,7 +11,7 @@ from functools import partial
 from typing import Any, List, Tuple
 
 from evalscope.constants import ArenaMode, EvalConfigKeys, FnCompletionParser, PositionBiasMitigation
-from evalscope.models.model import OpenAIModel
+from evalscope.models import OpenAIModel
 from evalscope.utils import completion_parsers, random_seeded_choice
 from evalscope.utils.arena_utils import get_battle_pairs, merge_ques_ans, shuffle_pairwise_preferences
 from evalscope.utils.io_utils import dump_jsonl_data, jsonl_to_list

evalscope/metrics/__init__.py

@@ -1,5 +1,50 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
-from evalscope.metrics.metrics import (bleu_ngram_one_sample, exact_match, macro_mean, mean, micro_mean,
-                                       simple_f1_score, weighted_mean)
-from evalscope.metrics.named_metrics import *
-from evalscope.metrics.rouge_metric import compute_rouge_score_one_sample_zh
+from typing import TYPE_CHECKING
+
+from evalscope.utils.import_utils import _LazyModule
+
+if TYPE_CHECKING:
+    from .llm_judge import LLMJudge
+    from .math_parser import extract_answer, math_equal, strip_answer_string
+    from .metrics import (bleu_ngram_one_sample, exact_match, macro_mean, mean, micro_mean, simple_f1_score,
+                          weighted_mean)
+    from .named_metrics import Metric, metric_registry
+    from .rouge_metric import compute_rouge_score_one_sample_zh
+
+else:
+    _import_structure = {
+        'metrics': [
+            'bleu_ngram_one_sample',
+            'exact_match',
+            'macro_mean',
+            'mean',
+            'micro_mean',
+            'simple_f1_score',
+            'weighted_mean',
+        ],
+        'named_metrics': [
+            'Metric',
+            'metric_registry',
+        ],
+        'rouge_metric': [
+            'compute_rouge_score_one_sample_zh',
+        ],
+        'llm_judge': [
+            'LLMJudge',
+        ],
+        'math_parser': [
+            'extract_answer',
+            'math_equal',
+            'strip_answer_string',
+        ],
+    }
+
+    import sys
+
+    sys.modules[__name__] = _LazyModule(
+        __name__,
+        globals()['__file__'],
+        _import_structure,
+        module_spec=__spec__,
+        extra_objects={},
+    )

evalscope/metrics/llm_judge.py

@@ -54,7 +54,7 @@ class LLMJudge:
         self.prompt_template = prompt_template or os.environ.get('JUDGE_PROMPT_TEMPLATE', DEFAULT_PROMPT_TEMPLATE)
         self.generation_config = generation_config
 
-        from evalscope.models.server_adapter import ServerModelAdapter
+        from evalscope.models import ServerModelAdapter
 
         # Initialize ServerModelAdapter
         self.server_adapter = ServerModelAdapter(api_url=self.api_url, model_id=self.model_id, api_key=self.api_key)

evalscope/metrics/named_metrics.py

@@ -3,6 +3,8 @@ from functools import partial
 from typing import Callable, Dict
 
 from evalscope.metrics.metrics import mean, pass_at_k, weighted_mean
+from evalscope.metrics.t2v_metrics import (blip2_score, clip_flant5_score, clip_score, fga_blip2_score, hpsv2_1_score,
+                                           hpsv2_score, image_reward_score, mps_score, pick_score)
 
 
 @dataclass
@@ -40,3 +42,14 @@ metric_registry.register(Metric(name='WeightedAverageBLEU', object=weighted_mean
 metric_registry.register(Metric(name='AveragePass@1', object=mean))
 for k in range(1, 17):
     metric_registry.register(Metric(name=f'Pass@{k}', object=partial(pass_at_k, k=k)))
+
+# t2v_metrics
+metric_registry.register(Metric(name='VQAScore', object=clip_flant5_score))
+metric_registry.register(Metric(name='PickScore', object=pick_score))
+metric_registry.register(Metric(name='CLIPScore', object=clip_score))
+metric_registry.register(Metric(name='BLIPv2Score', object=blip2_score))
+metric_registry.register(Metric(name='HPSv2Score', object=hpsv2_score))
+metric_registry.register(Metric(name='HPSv2.1Score', object=hpsv2_1_score))
+metric_registry.register(Metric(name='ImageRewardScore', object=image_reward_score))
+metric_registry.register(Metric(name='FGA_BLIP2Score', object=fga_blip2_score))
+metric_registry.register(Metric(name='MPS', object=mps_score))

evalscope/metrics/t2v_metrics/__init__.py

@@ -0,0 +1,66 @@
+from __future__ import absolute_import, division, print_function
+
+from .clipscore import CLIPScore, list_all_clipscore_models
+from .constants import CACHE_DIR
+from .itmscore import ITMScore, list_all_itmscore_models
+from .vqascore import VQAScore, list_all_vqascore_models
+
+
+def list_all_models():
+    return list_all_vqascore_models() + list_all_clipscore_models() + list_all_itmscore_models()
+
+
+def get_score_model(model='clip-flant5-xxl', device='cuda', cache_dir=CACHE_DIR, **kwargs):
+    if model in list_all_vqascore_models():
+        return VQAScore(model, device=device, cache_dir=cache_dir, **kwargs)
+    elif model in list_all_clipscore_models():
+        return CLIPScore(model, device=device, cache_dir=cache_dir, **kwargs)
+    elif model in list_all_itmscore_models():
+        return ITMScore(model, device=device, cache_dir=cache_dir, **kwargs)
+    else:
+        raise NotImplementedError()
+
+
+def clip_flant5_score():
+    clip_flant5_score = VQAScore(model='clip-flant5-xxl')
+    return clip_flant5_score
+
+
+def pick_score():
+    pick_score = CLIPScore(model='pickscore-v1')
+    return pick_score
+
+
+def clip_score():
+    clip_score = CLIPScore(model='openai:ViT-L-14-336')
+    return clip_score
+
+
+def blip2_score():
+    blip_itm_score = ITMScore(model='blip2-itm')
+    return blip_itm_score
+
+
+def hpsv2_score():
+    hpsv2_score = CLIPScore(model='hpsv2')
+    return hpsv2_score
+
+
+def hpsv2_1_score():
+    hpsv2_1_score = CLIPScore(model='hpsv2.1')
+    return hpsv2_1_score
+
+
+def image_reward_score():
+    image_reward_score = ITMScore(model='image-reward-v1')
+    return image_reward_score
+
+
+def fga_blip2_score():
+    fga_blip2_score = ITMScore(model='fga_blip2')
+    return fga_blip2_score
+
+
+def mps_score():
+    mps_score = CLIPScore(model='mps')
+    return mps_score

evalscope/metrics/t2v_metrics/clipscore.py

@@ -0,0 +1,14 @@
+from typing import List
+
+from .constants import CACHE_DIR
+from .models.clipscore_models import get_clipscore_model, list_all_clipscore_models
+from .score import Score
+
+
+class CLIPScore(Score):
+
+    def prepare_scoremodel(self, model='openai:ViT-L/14', device='cuda', cache_dir=CACHE_DIR):
+        return get_clipscore_model(model, device=device, cache_dir=cache_dir)
+
+    def list_all_models(self) -> List[str]:
+        return list_all_clipscore_models()

evalscope/metrics/t2v_metrics/constants.py

@@ -0,0 +1,12 @@
+import os
+from modelscope.utils.file_utils import get_model_cache_root
+
+CACHE_DIR = get_model_cache_root()
+os.environ['TORCH_HOME'] = CACHE_DIR  # set timm cache dir
+
+# For CLIP-FlanT5
+CONTEXT_LEN = 2048
+SYSTEM_MSG = "A chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions."
+IGNORE_INDEX = -100
+IMAGE_TOKEN_INDEX = -200
+DEFAULT_IMAGE_TOKEN = '<image>'

evalscope/metrics/t2v_metrics/itmscore.py

@@ -0,0 +1,14 @@
+from typing import List
+
+from .constants import CACHE_DIR
+from .models.itmscore_models import get_itmscore_model, list_all_itmscore_models
+from .score import Score
+
+
+class ITMScore(Score):
+
+    def prepare_scoremodel(self, model='blip2-itm', device='cuda', cache_dir=CACHE_DIR):
+        return get_itmscore_model(model, device=device, cache_dir=cache_dir)
+
+    def list_all_models(self) -> List[str]:
+        return list_all_itmscore_models()

evalscope/metrics/t2v_metrics/models/clipscore_models/__init__.py

@@ -0,0 +1,30 @@
+from ...constants import CACHE_DIR
+from .clip_model import CLIP_MODELS, CLIPScoreModel
+from .hpsv2_model import HPSV2_MODELS, HPSV2ScoreModel
+from .mps_model import MPS_MODELS, MPSModel
+from .pickscore_model import PICKSCORE_MODELS, PickScoreModel
+
+ALL_CLIP_MODELS = [
+    CLIP_MODELS,
+    HPSV2_MODELS,
+    PICKSCORE_MODELS,
+    MPS_MODELS,
+]
+
+
+def list_all_clipscore_models():
+    return [model for models in ALL_CLIP_MODELS for model in models]
+
+
+def get_clipscore_model(model_name, device='cuda', cache_dir=CACHE_DIR):
+    assert model_name in list_all_clipscore_models()
+    if model_name in CLIP_MODELS:
+        return CLIPScoreModel(model_name, device=device, cache_dir=cache_dir)
+    elif model_name in HPSV2_MODELS:
+        return HPSV2ScoreModel(model_name, device=device, cache_dir=cache_dir)
+    elif model_name in PICKSCORE_MODELS:
+        return PickScoreModel(model_name, device=device, cache_dir=cache_dir)
+    elif model_name in MPS_MODELS:
+        return MPSModel(model_name, device=device, cache_dir=cache_dir)
+    else:
+        raise NotImplementedError()

evalscope/metrics/t2v_metrics/models/clipscore_models/build_mps_model/base_model.py

@@ -0,0 +1,6 @@
+from dataclasses import dataclass
+
+
+@dataclass
+class BaseModelConfig:
+    pass