evalscope 0.13.2__py3-none-any.whl → 0.15.0__py3-none-any.whl

evalscope/arguments.py

@@ -1,7 +1,7 @@
 import argparse
 import json
 
-from evalscope.constants import EvalBackend, EvalStage, EvalType, JudgeStrategy, OutputType
+from evalscope.constants import EvalBackend, EvalStage, EvalType, JudgeStrategy, ModelTask, OutputType
 
 
 class ParseStrArgsAction(argparse.Action):
@@ -35,6 +35,7 @@ def add_argument(parser: argparse.ArgumentParser):
     parser.add_argument('--model', type=str, required=False, help='The model id on modelscope, or local model dir.')
     parser.add_argument('--model-id', type=str, required=False, help='The model id for model name in report.')
     parser.add_argument('--model-args', type=str, action=ParseStrArgsAction, help='The model args, should be a string.')
+    parser.add_argument('--model-task', type=str, default=ModelTask.TEXT_GENERATION, choices=[ModelTask.TEXT_GENERATION, ModelTask.IMAGE_GENERATION], help='The model task for model id.')  # noqa: E501
 
     # Template-related arguments
     parser.add_argument('--template-type', type=str, required=False, help='Deprecated, will be removed in v1.0.0.')

evalscope/backend/rag_eval/__init__.py

@@ -1,4 +1,4 @@
-from evalscope.backend.rag_eval.backend_manager import RAGEvalBackendManager
+from evalscope.backend.rag_eval.backend_manager import RAGEvalBackendManager, Tools
 from evalscope.backend.rag_eval.utils.clip import VisionModel
 from evalscope.backend.rag_eval.utils.embedding import EmbeddingModel
 from evalscope.backend.rag_eval.utils.llm import LLM, ChatOpenAI, LocalLLM

evalscope/backend/rag_eval/backend_manager.py

@@ -8,6 +8,12 @@ from evalscope.utils.logger import get_logger
 logger = get_logger()
 
 
+class Tools:
+    MTEB = 'mteb'
+    RAGAS = 'ragas'
+    CLIP_BENCHMARK = 'clip_benchmark'
+
+
 class RAGEvalBackendManager(BackendManager):
 
     def __init__(self, config: Union[str, dict], **kwargs):
@@ -47,9 +53,19 @@ class RAGEvalBackendManager(BackendManager):
         from evalscope.backend.rag_eval.ragas.tasks import generate_testset
 
         if testset_args is not None:
-            generate_testset(TestsetGenerationArguments(**testset_args))
+            if isinstance(testset_args, dict):
+                generate_testset(TestsetGenerationArguments(**testset_args))
+            elif isinstance(testset_args, TestsetGenerationArguments):
+                generate_testset(testset_args)
+            else:
+                raise ValueError('Please provide the testset generation arguments.')
         if eval_args is not None:
-            rag_eval(EvaluationArguments(**eval_args))
+            if isinstance(eval_args, dict):
+                rag_eval(EvaluationArguments(**eval_args))
+            elif isinstance(eval_args, EvaluationArguments):
+                rag_eval(eval_args)
+            else:
+                raise ValueError('Please provide the evaluation arguments.')
 
     @staticmethod
     def run_clip_benchmark(args):
@@ -59,17 +75,17 @@ class RAGEvalBackendManager(BackendManager):
 
     def run(self, *args, **kwargs):
         tool = self.config_d.pop('tool')
-        if tool.lower() == 'mteb':
+        if tool.lower() == Tools.MTEB:
             self._check_env('mteb')
             model_args = self.config_d['model']
             eval_args = self.config_d['eval']
             self.run_mteb(model_args, eval_args)
-        elif tool.lower() == 'ragas':
+        elif tool.lower() == Tools.RAGAS:
             self._check_env('ragas')
             testset_args = self.config_d.get('testset_generation', None)
             eval_args = self.config_d.get('eval', None)
             self.run_ragas(testset_args, eval_args)
-        elif tool.lower() == 'clip_benchmark':
+        elif tool.lower() == Tools.CLIP_BENCHMARK:
             self._check_env('webdataset')
             self.run_clip_benchmark(self.config_d['eval'])
         else:

evalscope/backend/rag_eval/cmteb/arguments.py

@@ -20,6 +20,12 @@ class ModelArguments:
     encode_kwargs: dict = field(default_factory=lambda: {'show_progress_bar': True, 'batch_size': 32})
     hub: str = 'modelscope'  # modelscope or huggingface
 
+    # for API embedding model
+    model_name: Optional[str] = None
+    api_base: Optional[str] = None
+    api_key: Optional[str] = None
+    dimensions: Optional[int] = None
+
     def to_dict(self) -> Dict[str, Any]:
         return {
             'model_name_or_path': self.model_name_or_path,
@@ -31,6 +37,10 @@ class ModelArguments:
             'config_kwargs': self.config_kwargs,
             'encode_kwargs': self.encode_kwargs,
             'hub': self.hub,
+            'model_name': self.model_name,
+            'api_base': self.api_base,
+            'api_key': self.api_key,
+            'dimensions': self.dimensions,
         }
 
 

evalscope/backend/rag_eval/ragas/arguments.py

@@ -21,7 +21,6 @@ class TestsetGenerationArguments:
     """
     generator_llm: Dict = field(default_factory=dict)
     embeddings: Dict = field(default_factory=dict)
-    distribution: str = field(default_factory=lambda: {'simple': 0.5, 'multi_context': 0.4, 'reasoning': 0.1})
     # For LLM based evaluation
     # available: ['english', 'hindi', 'marathi', 'chinese', 'spanish', 'amharic', 'arabic',
     # 'armenian', 'bulgarian', 'urdu', 'russian', 'polish', 'persian', 'dutch', 'danish',

evalscope/backend/rag_eval/ragas/tasks/testset_generation.py

@@ -67,9 +67,14 @@ def get_persona(llm, kg, language):
 
 
 def load_data(file_path):
-    from langchain_community.document_loaders import UnstructuredFileLoader
+    import nltk
+    from langchain_unstructured import UnstructuredLoader
 
-    loader = UnstructuredFileLoader(file_path, mode='single')
+    if nltk.data.find('taggers/averaged_perceptron_tagger_eng') is False:
+        # need to download nltk data for the first time
+        nltk.download('averaged_perceptron_tagger_eng')
+
+    loader = UnstructuredLoader(file_path)
     data = loader.load()
     return data
 

evalscope/backend/rag_eval/ragas/tasks/translate_prompt.py

@@ -2,7 +2,6 @@ import asyncio
 import os
 from ragas.llms import BaseRagasLLM
 from ragas.prompt import PromptMixin, PydanticPrompt
-from ragas.utils import RAGAS_SUPPORTED_LANGUAGE_CODES
 from typing import List
 
 from evalscope.utils.logger import get_logger
@@ -16,10 +15,6 @@ async def translate_prompt(
     llm: BaseRagasLLM,
     adapt_instruction: bool = False,
 ):
-    if target_lang not in RAGAS_SUPPORTED_LANGUAGE_CODES:
-        logger.warning(f'{target_lang} is not in supported language: {list(RAGAS_SUPPORTED_LANGUAGE_CODES)}')
-        return
-
     if not issubclass(type(prompt_user), PromptMixin):
         logger.info(f"{prompt_user} is not a PromptMixin, don't translate it")
         return

evalscope/backend/rag_eval/utils/embedding.py

@@ -1,10 +1,12 @@
 import os
 import torch
 from langchain_core.embeddings import Embeddings
+from langchain_openai.embeddings import OpenAIEmbeddings
 from sentence_transformers import models
 from sentence_transformers.cross_encoder import CrossEncoder
 from sentence_transformers.SentenceTransformer import SentenceTransformer
 from torch import Tensor
+from tqdm import tqdm
 from typing import Dict, List, Optional, Union
 
 from evalscope.backend.rag_eval.utils.tools import download_model
@@ -18,10 +20,10 @@ class BaseModel(Embeddings):
 
     def __init__(
         self,
-        model_name_or_path: str,
+        model_name_or_path: str = '',
         max_seq_length: int = 512,
         prompt: str = '',
-        revision: Optional[str] = None,
+        revision: Optional[str] = 'master',
         **kwargs,
     ):
         self.model_name_or_path = model_name_or_path
@@ -139,7 +141,7 @@ class CrossEncoderModel(BaseModel):
             max_length=self.max_seq_length,
         )
 
-    def predict(self, sentences: List[List[str]], **kwargs) -> List[List[float]]:
+    def predict(self, sentences: List[List[str]], **kwargs) -> Tensor:
         self.encode_kwargs.update(kwargs)
 
         if len(sentences[0]) == 3:  # Note: For mteb retrieval task
@@ -154,6 +156,46 @@ class CrossEncoderModel(BaseModel):
         return embeddings
 
 
+class APIEmbeddingModel(BaseModel):
+
+    def __init__(self, **kwargs):
+        self.model_name = kwargs.get('model_name')
+        self.openai_api_base = kwargs.get('api_base')
+        self.openai_api_key = kwargs.get('api_key')
+        self.dimensions = kwargs.get('dimensions')
+
+        self.model = OpenAIEmbeddings(
+            model=self.model_name,
+            openai_api_base=self.openai_api_base,
+            openai_api_key=self.openai_api_key,
+            dimensions=self.dimensions,
+            check_embedding_ctx_length=False)
+
+        super().__init__(model_name_or_path=self.model_name, **kwargs)
+
+        self.batch_size = self.encode_kwargs.get('batch_size', 10)
+
+    def encode(self, texts: Union[str, List[str]], **kwargs) -> Tensor:
+        if isinstance(texts, str):
+            texts = [texts]
+
+        embeddings: List[List[float]] = []
+        for i in tqdm(range(0, len(texts), self.batch_size)):
+            response = self.model.embed_documents(texts[i:i + self.batch_size], chunk_size=self.batch_size)
+            embeddings.extend(response)
+        return torch.tensor(embeddings)
+
+    def encode_queries(self, queries, **kwargs):
+        return self.encode(queries, **kwargs)
+
+    def encode_corpus(self, corpus, **kwargs):
+        if isinstance(corpus[0], dict):
+            input_texts = ['{} {}'.format(doc.get('title', ''), doc['text']).strip() for doc in corpus]
+        else:
+            input_texts = corpus
+        return self.encode(input_texts, **kwargs)
+
+
 class EmbeddingModel:
     """Custom embeddings"""
 
@@ -165,6 +207,10 @@ class EmbeddingModel:
         revision: Optional[str] = 'master',
         **kwargs,
     ):
+        if kwargs.get('model_name'):
+            # If model_name is provided, use OpenAIEmbeddings
+            return APIEmbeddingModel(**kwargs)
+
         # If model path does not exist and hub is 'modelscope', download the model
         if not os.path.exists(model_name_or_path) and hub == HubType.MODELSCOPE:
             model_name_or_path = download_model(model_name_or_path, revision)

evalscope/backend/rag_eval/utils/llm.py

@@ -2,7 +2,7 @@ import os
 from langchain_core.callbacks.manager import CallbackManagerForLLMRun
 from langchain_core.language_models.llms import LLM as BaseLLM
 from langchain_openai import ChatOpenAI
-from modelscope.utils.hf_util import GenerationConfig
+from transformers.generation.configuration_utils import GenerationConfig
 from typing import Any, Dict, Iterator, List, Mapping, Optional
 
 from evalscope.constants import DEFAULT_MODEL_REVISION
@@ -16,9 +16,9 @@ class LLM:
         api_base = kw.get('api_base', None)
         if api_base:
             return ChatOpenAI(
-                model_name=kw.get('model_name', ''),
-                openai_api_base=api_base,
-                openai_api_key=kw.get('api_key', 'EMPTY'),
+                model=kw.get('model_name', ''),
+                base_url=api_base,
+                api_key=kw.get('api_key', 'EMPTY'),
             )
         else:
             return LocalLLM(**kw)

evalscope/backend/vlm_eval_kit/backend_manager.py

@@ -1,4 +1,5 @@
 import copy
+import os
 import subprocess
 from functools import partial
 from typing import Optional, Union
@@ -66,8 +67,9 @@ class VLMEvalKitBackendManager(BackendManager):
                     del remain_cfg['name']  # remove not used args
                     del remain_cfg['type']  # remove not used args
 
-                    self.valid_models.update({model_type: partial(model_class, model=model_type, **remain_cfg)})
-                    new_model_names.append(model_type)
+                    norm_model_type = os.path.basename(model_type).replace(':', '-').replace('.', '_')
+                    self.valid_models.update({norm_model_type: partial(model_class, model=model_type, **remain_cfg)})
+                    new_model_names.append(norm_model_type)
                 else:
                     remain_cfg = copy.deepcopy(model_cfg)
                     del remain_cfg['name']  # remove not used args

evalscope/benchmarks/__init__.py

@@ -10,8 +10,8 @@ from evalscope.utils import get_logger
 logger = get_logger()
 
 # Using glob to find all files matching the pattern
-pattern = os.path.join(os.path.dirname(__file__), '*', '*_adapter.py')
-files = glob.glob(pattern, recursive=False)
+pattern = os.path.join(os.path.dirname(__file__), '*', '**', '*_adapter.py')
+files = glob.glob(pattern, recursive=True)
 
 for file_path in files:
     if file_path.endswith('.py') and not os.path.basename(file_path).startswith('_'):

evalscope/benchmarks/aigc/t2i/base.py

@@ -0,0 +1,56 @@
+from typing import List, Optional, Union
+
+from evalscope.benchmarks import DataAdapter
+from evalscope.metrics import mean, metric_registry
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+
+class T2IBaseAdapter(DataAdapter):
+
+    def __init__(self, **kwargs):
+
+        super().__init__(**kwargs)
+
+        logger.info(f'Initializing metrics: {self.metric_list}')
+        self.metrics = {m: metric_registry.get(m).object() for m in self.metric_list}
+
+    def gen_prompt(self, input_d: dict, subset_name: str, few_shot_list: list, **kwargs) -> dict:
+        # dummy prompt for general t2i
+        return self.gen_prompt_data(prompt=input_d.get('prompt', ''), id=input_d.get('id', 0))
+
+    def get_gold_answer(self, input_d: dict) -> str:
+        # dummy gold answer for general t2i
+        return input_d.get('prompt', '')
+
+    def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = 'checkpoint') -> str:
+        # dummy parse pred result for general t2i
+        return result or raw_input_d.get('image_path', '')
+
+    def match(self, gold: str, pred: str) -> dict:
+        # dummy match for general t2i
+        # pred is the image path, gold is the prompt
+        res = {}
+        for metric_name, metric_func in self.metrics.items():
+            score = metric_func(images=[pred], texts=[gold])[0][0]
+            if isinstance(score, dict):
+                for k, v in score.items():
+                    res[f'{metric_name}_{k}'] = v.cpu().item()
+            else:
+                res[metric_name] = score.cpu().item()  # Updated to use score.cpu().item()
+        return res
+
+    def compute_metric(self, review_res_list: Union[List[dict], List[List[dict]]], **kwargs) -> List[dict]:
+        """
+        compute weighted mean of the bleu score of all samples
+
+        Args:
+            review_res_list: [score1, score2, ...]
+
+        Returns:
+            avg_res: List[dict]
+
+        """
+        items = super().compute_dict_metric(review_res_list, **kwargs)
+        return [{'metric_name': k, 'score': mean(v), 'num': len(v)} for k, v in items.items()]

evalscope/benchmarks/aigc/t2i/evalmuse_adapter.py

@@ -0,0 +1,77 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import os.path
+from collections import defaultdict
+from typing import List, Optional, Union
+
+from evalscope.benchmarks import Benchmark
+from evalscope.constants import OutputType
+from evalscope.metrics import mean
+from evalscope.utils.io_utils import jsonl_to_list
+from evalscope.utils.logger import get_logger
+from .base import T2IBaseAdapter
+
+logger = get_logger()
+
+
+@Benchmark.register(
+    name='evalmuse',
+    dataset_id='AI-ModelScope/T2V-Eval-Prompts',
+    model_adapter=OutputType.IMAGE_GENERATION,
+    output_types=[OutputType.IMAGE_GENERATION],
+    subset_list=['EvalMuse'],
+    metric_list=['FGA_BLIP2Score'],
+    few_shot_num=0,
+    train_split=None,
+    eval_split='test',
+)
+class EvalMuseAdapter(T2IBaseAdapter):
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    def load(self, **kwargs) -> dict:
+        if os.path.isfile(self.dataset_id):
+            data_list = jsonl_to_list(self.dataset_id)
+            data_dict = {self.subset_list[0]: {'test': data_list}}
+            return data_dict
+        else:
+            return super().load(**kwargs)
+
+    def get_gold_answer(self, input_d: dict) -> dict:
+        # return prompt and elements dict
+        return {'prompt': input_d.get('prompt'), 'tags': input_d.get('tags', {})}
+
+    def match(self, gold: dict, pred: str) -> dict:
+        # dummy match for general t2i
+        # pred is the image path, gold is the prompt
+        res = {}
+        for metric_name, metric_func in self.metrics.items():
+            if metric_name == 'FGA_BLIP2Score':
+                # For FGA_BLIP2Score, we need to pass the dictionary
+                score = metric_func(images=[pred], texts=[gold])[0][0]
+            else:
+                score = metric_func(images=[pred], texts=[gold['prompt']])[0][0]
+            if isinstance(score, dict):
+                for k, v in score.items():
+                    res[f'{metric_name}:{k}'] = v.cpu().item()
+            else:
+                res[metric_name] = score.cpu().item()
+        return res
+
+    def compute_metric(self, review_res_list: Union[List[dict], List[List[dict]]], **kwargs) -> List[dict]:
+        """
+        compute weighted mean of the bleu score of all samples
+        """
+        items = super().compute_dict_metric(review_res_list, **kwargs)
+        # add statistics for each metric
+        new_items = defaultdict(list)
+        for metric_name, value_list in items.items():
+            if 'FGA_BLIP2Score' in metric_name and '(' in metric_name:  # FGA_BLIP2Score element score
+                metrics_prefix = metric_name.split(':')[0]
+                category = metric_name.rpartition('(')[-1].split(')')[0]
+                new_items[f'{metrics_prefix}:{category}'].extend(value_list)
+            else:
+                new_items[metric_name].extend(value_list)
+
+        # calculate mean for each metric
+        return [{'metric_name': k, 'score': mean(v), 'num': len(v)} for k, v in new_items.items()]

evalscope/benchmarks/aigc/t2i/genai_bench_adapter.py

@@ -0,0 +1,58 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import os.path
+from collections import defaultdict
+from typing import List, Optional, Union
+
+from evalscope.benchmarks import Benchmark
+from evalscope.constants import OutputType
+from evalscope.utils.io_utils import jsonl_to_list
+from evalscope.utils.logger import get_logger
+from .base import T2IBaseAdapter
+
+logger = get_logger()
+
+
+@Benchmark.register(
+    name='genai_bench',
+    dataset_id='AI-ModelScope/T2V-Eval-Prompts',
+    model_adapter=OutputType.IMAGE_GENERATION,
+    output_types=[OutputType.IMAGE_GENERATION],
+    subset_list=['GenAI-Bench-1600'],
+    metric_list=['VQAScore'],
+    few_shot_num=0,
+    train_split=None,
+    eval_split='test',
+)
+class GenAIBenchAdapter(T2IBaseAdapter):
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    def load(self, **kwargs) -> dict:
+        if os.path.isfile(self.dataset_id):
+            data_list = jsonl_to_list(self.dataset_id)
+            data_dict = {self.subset_list[0]: {'test': data_list}}
+            return data_dict
+        else:
+            return super().load(**kwargs)
+
+    def get_gold_answer(self, input_d: dict) -> dict:
+        # return prompt and elements dict
+        return {'prompt': input_d.get('prompt'), 'tags': input_d.get('tags', {})}
+
+    def match(self, gold: dict, pred: str) -> dict:
+        # dummy match for general t2i
+        # pred is the image path, gold is the prompt
+        res = {}
+        for metric_name, metric_func in self.metrics.items():
+            score = metric_func(images=[pred], texts=[gold['prompt']])[0][0]
+
+            res[metric_name] = score.cpu().item()
+
+            # fine-granular metrics
+            if gold['tags'].get('advanced'):
+                res[f'{metric_name}_advanced'] = score.cpu().item()
+            else:
+                res[f'{metric_name}_basic'] = score.cpu().item()
+
+        return res

evalscope/benchmarks/aigc/t2i/general_t2i_adapter.py

@@ -0,0 +1,58 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import os.path
+from collections import defaultdict
+from typing import List, Optional, Union
+
+from evalscope.benchmarks import Benchmark
+from evalscope.constants import OutputType
+from evalscope.utils.io_utils import jsonl_to_list
+from evalscope.utils.logger import get_logger
+from .base import T2IBaseAdapter
+
+logger = get_logger()
+
+
+@Benchmark.register(
+    name='general_t2i',
+    dataset_id='general_t2i',
+    model_adapter=OutputType.IMAGE_GENERATION,
+    output_types=[OutputType.IMAGE_GENERATION],
+    subset_list=['default'],
+    metric_list=['PickScore'],
+    few_shot_num=0,
+    train_split=None,
+    eval_split='test',
+)
+class GeneralT2IAdapter(T2IBaseAdapter):
+
+    def __init__(self, **kwargs):
+
+        super().__init__(**kwargs)
+
+    def load(self, dataset_name_or_path: str = None, subset_list: list = None, **kwargs) -> dict:
+        dataset_name_or_path = dataset_name_or_path or self.dataset_id
+        subset_list = subset_list or self.subset_list
+
+        data_file_dict = defaultdict(str)
+        data_list = []
+
+        # get data file path and subset name
+        if os.path.isdir(dataset_name_or_path):
+            for subset_name in subset_list:
+                data_file_dict[subset_name] = os.path.join(dataset_name_or_path, f'{subset_name}.jsonl')
+        elif os.path.isfile(dataset_name_or_path):
+            cur_subset_name = os.path.splitext(os.path.basename(dataset_name_or_path))[0]
+            data_file_dict[cur_subset_name] = dataset_name_or_path
+        else:
+            raise ValueError(f'Invalid dataset path: {dataset_name_or_path}')
+
+        # load data from local disk
+        try:
+            for subset_name, file_path in data_file_dict.items():
+                data_list.extend(jsonl_to_list(file_path))
+        except Exception as e:
+            raise ValueError(f'Failed to load data from {self.dataset_id}, got error: {e}')
+
+        data_dict = {subset_name: {'test': data_list} for subset_name in data_file_dict.keys()}
+
+        return data_dict

evalscope/benchmarks/aigc/t2i/hpdv2_adapter.py

@@ -0,0 +1,57 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import os.path
+from collections import defaultdict
+from typing import List, Optional, Union
+
+from evalscope.benchmarks import Benchmark
+from evalscope.constants import OutputType
+from evalscope.utils.io_utils import jsonl_to_list
+from evalscope.utils.logger import get_logger
+from .base import T2IBaseAdapter
+
+logger = get_logger()
+
+
+@Benchmark.register(
+    name='hpdv2',
+    dataset_id='AI-ModelScope/T2V-Eval-Prompts',
+    model_adapter=OutputType.IMAGE_GENERATION,
+    output_types=[OutputType.IMAGE_GENERATION],
+    subset_list=['HPDv2'],
+    metric_list=['HPSv2.1Score'],
+    few_shot_num=0,
+    train_split=None,
+    eval_split='test',
+)
+class HPDv2Adapter(T2IBaseAdapter):
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    def load(self, **kwargs) -> dict:
+        if os.path.isfile(self.dataset_id):
+            data_list = jsonl_to_list(self.dataset_id)
+            data_dict = {self.subset_list[0]: {'test': data_list}}
+            return data_dict
+        else:
+            return super().load(**kwargs)
+
+    def get_gold_answer(self, input_d: dict) -> dict:
+        # return prompt and elements dict
+        return {'prompt': input_d.get('prompt'), 'tags': input_d.get('tags', {})}
+
+    def match(self, gold: dict, pred: str) -> dict:
+        # dummy match for general t2i
+        # pred is the image path, gold is the prompt
+        res = {}
+        for metric_name, metric_func in self.metrics.items():
+            score = metric_func(images=[pred], texts=[gold['prompt']])[0][0]
+
+            res[metric_name] = score.cpu().item()
+
+            # fine-granular metrics
+            category = gold['tags'].get('category')
+            if category:
+                res[f'{metric_name}_{category}'] = score.cpu().item()
+
+        return res

evalscope/benchmarks/aigc/t2i/tifa_adapter.py

@@ -0,0 +1,37 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import os.path
+from collections import defaultdict
+from typing import List, Optional, Union
+
+from evalscope.benchmarks import Benchmark
+from evalscope.constants import OutputType
+from evalscope.utils.io_utils import jsonl_to_list
+from evalscope.utils.logger import get_logger
+from .base import T2IBaseAdapter
+
+logger = get_logger()
+
+
+@Benchmark.register(
+    name='tifa160',
+    dataset_id='AI-ModelScope/T2V-Eval-Prompts',
+    model_adapter=OutputType.IMAGE_GENERATION,
+    output_types=[OutputType.IMAGE_GENERATION],
+    subset_list=['TIFA-160'],
+    metric_list=['PickScore'],
+    few_shot_num=0,
+    train_split=None,
+    eval_split='test',
+)
+class TIFA_Adapter(T2IBaseAdapter):
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    def load(self, **kwargs) -> dict:
+        if os.path.isfile(self.dataset_id):
+            data_list = jsonl_to_list(self.dataset_id)
+            data_dict = {self.subset_list[0]: {'test': data_list}}
+            return data_dict
+        else:
+            return super().load(**kwargs)

evalscope/benchmarks/aime/aime24_adapter.py

@@ -1,6 +1,6 @@
 from evalscope.benchmarks import Benchmark, DataAdapter
 from evalscope.constants import OutputType
-from evalscope.metrics.math_parser import extract_answer, math_equal, strip_answer_string
+from evalscope.metrics import extract_answer, math_equal, strip_answer_string
 from evalscope.utils.logger import get_logger
 
 # flake8: noqa