evalscope 0.11.0__py3-none-any.whl → 0.12.1__py3-none-any.whl

evalscope/benchmarks/data_adapter.py

@@ -2,8 +2,10 @@
 import os.path
 import random
 from abc import ABC, abstractmethod
+from collections import defaultdict
 from typing import Any, List, Optional, Union
 
+from evalscope.benchmarks.utils import PromptData, preprocess_decorator
 from evalscope.constants import DEFAULT_DATASET_CACHE_DIR, AnswerKeys, EvalType, HubType
 from evalscope.metrics.named_metrics import metric_registry
 from evalscope.report import Report, ReportGenerator
@@ -17,6 +19,7 @@ class DataAdapter(ABC):
     def __init__(self,
                  name: str,
                  dataset_id: str,
+                 model_adapter: str,
                  subset_list: list,
                  metric_list: List[str],
                  few_shot_num: Optional[int] = 0,
@@ -24,6 +27,8 @@ class DataAdapter(ABC):
                  eval_split: Optional[str] = None,
                  prompt_template: Optional[str] = None,
                  system_prompt: Optional[str] = None,
+                 query_template: Optional[str] = None,
+                 pretty_name: Optional[str] = None,
                  **kwargs):
         """
         Data Adapter for the benchmark. You need to implement the following methods:
@@ -45,6 +50,7 @@ class DataAdapter(ABC):
         """
         self.name = name
         self.dataset_id = dataset_id
+        self.model_adapter = model_adapter
         self.subset_list = subset_list
         self.metric_list = metric_list
         self.few_shot_num = few_shot_num
@@ -52,14 +58,24 @@ class DataAdapter(ABC):
         self.eval_split = eval_split
         self.prompt_template = prompt_template
         self.system_prompt = system_prompt
+        self.query_template = query_template
+        self.pretty_name = pretty_name
         self.config_kwargs = kwargs
         self.category_map = kwargs.get('category_map', {})
+        self.choices = kwargs.get('choices', None)
+
+    def __init_subclass__(cls, **kwargs):
+        super().__init_subclass__(**kwargs)
+
+        # find and decorate parse_pred_result method
+        if hasattr(cls, 'parse_pred_result'):
+            original_method = cls.parse_pred_result
+            cls.parse_pred_result = preprocess_decorator(original_method)
 
     def load(self,
              dataset_name_or_path: str = None,
              subset_list: list = None,
              work_dir: Optional[str] = DEFAULT_DATASET_CACHE_DIR,
-             datasets_hub: str = HubType.MODELSCOPE,
              **kwargs) -> dict:
         """
         Load the dataset. Remote and local datasets are supported.
@@ -74,22 +90,43 @@ class DataAdapter(ABC):
 
         # Try to load dataset from local disk
         if os.path.exists(dataset_name_or_path):
-            logger.info(f'Loading dataset from work_dir: {work_dir}: > dataset_name: {dataset_name_or_path} > \
-                    subsets: {subset_list}')
-            data_dict = self.load_from_disk(dataset_name_or_path, subset_list, work_dir, **kwargs)
-            if len(data_dict) == 0 or len(next(iter(data_dict.values()))) == 0:
-                raise ValueError(f'Local dataset is empty: {dataset_name_or_path}')
+            logger.info(f'Loading dataset from local disk: {dataset_name_or_path}')
+            data_dict = self.load_from_disk(
+                dataset_name_or_path, subset_list, work_dir, trust_remote_code=False, **kwargs)
         else:
-            from modelscope.msdatasets import MsDataset
+            logger.info(f'Loading dataset from hub: {dataset_name_or_path}')
+            data_dict = self.load_from_hub(
+                dataset_name_or_path, subset_list, work_dir, trust_remote_code=True, **kwargs)
+        if len(data_dict) == 0:
+            raise ValueError(f'Dataset is empty: {dataset_name_or_path}')
+        return data_dict
+
+    def load_from_hub(self, dataset_name_or_path: str, subset_list: list, work_dir: str, **kwargs) -> dict:
+        from modelscope.msdatasets import MsDataset
 
-            # Load dataset from remote
-            logger.info(
-                f'Loading dataset from {datasets_hub}: > dataset_name: {dataset_name_or_path} > subsets: {subset_list}')
-            data_dict = {}
-            split_list = [split for split in [self.train_split, self.eval_split] if split is not None]
-            if len(split_list) == 0:
-                logger.error(f'Got empty split list: {split_list}')
+        datasets_hub: str = kwargs.pop('datasets_hub', HubType.MODELSCOPE)
+        split_as_subset: bool = kwargs.pop('split_as_subset', False)
+        # Load dataset from remote
+        logger.info(f'Loading dataset: dataset_name: {dataset_name_or_path} > subsets: {subset_list}')
 
+        data_dict = {}
+        split_list = [split for split in [self.train_split, self.eval_split] if split is not None]
+        if len(split_list) == 0:
+            logger.error(f'Got empty split list: {split_list}')
+
+        if split_as_subset:
+            for sub_name in subset_list:
+                data_dict[sub_name] = {}
+                # e.g. train: few-shot, test: target dataset to evaluate
+                for split in split_list:
+                    dataset = MsDataset.load(
+                        dataset_name=dataset_name_or_path,
+                        split=sub_name,  # load subset from split
+                        cache_dir=work_dir,
+                        hub=datasets_hub,
+                        **kwargs)
+                    data_dict[sub_name].update({split: dataset})
+        else:
             for sub_name in subset_list:
                 data_dict[sub_name] = {}
                 # e.g. train: few-shot, test: target dataset to evaluate
@@ -101,17 +138,34 @@ class DataAdapter(ABC):
                         cache_dir=work_dir,
                         hub=datasets_hub,
                         **kwargs)
-
                     data_dict[sub_name].update({split: dataset})
 
         return data_dict
 
-    def load_from_disk(self, *args, **kwargs) -> dict:
+    def load_from_disk(self, dataset_name_or_path, subset_list, work_dir, **kwargs) -> dict:
         """
         Load the dataset from local disk.
         If you want to support local dataset, please rewrite this method in xxx_data_adapter.
+        Use modelscope.msdatasets.MsDataset.load to load the dataset from local by default.
+        """
+        return self.load_from_hub(dataset_name_or_path, subset_list, work_dir, **kwargs)
+
+    def reformat_subset(self, data_dict: dict, subset_key: str, format: str = '{}') -> dict:
         """
-        return {}
+        Reformat the dataset subset with subset_key and format.
+        """
+        res_dict: dict = defaultdict(lambda: defaultdict(list), {key: defaultdict(list) for key in self.subset_list})
+
+        for sub_name, sub_data_dict in data_dict.items():
+            for split in [self.train_split, self.eval_split]:
+                if split is None:
+                    continue
+                for sample_d in sub_data_dict[split]:
+                    new_subset_name = format.format(sample_d[subset_key])
+                    if new_subset_name not in self.subset_list:
+                        continue
+                    res_dict[new_subset_name][split].append(sample_d)
+        return res_dict
 
     def gen_prompts(self, data_dict: dict) -> dict:
         """
@@ -138,7 +192,7 @@ class DataAdapter(ABC):
 
         for sub_name, sub_data_dict in data_dict.items():
             few_shot_data = []
-            if self.few_shot_num and self.few_shot_num > 0:
+            if self.train_split and self.few_shot_num and self.few_shot_num > 0:
                 few_shot_random: bool = self.config_kwargs.get('few_shot_random', True)
                 few_shot_data = self.get_fewshot_examples([item for item in sub_data_dict[self.train_split]],
                                                           self.few_shot_num,
@@ -161,7 +215,7 @@ class DataAdapter(ABC):
         else:
             return data_list[:k]
 
-    def compute_metric(self, review_res_list: Union[dict, list]) -> List[dict]:
+    def compute_metric(self, review_res_list: Union[dict, list], **kwargs) -> List[dict]:
         """
         Compute evaluation result by specific metrics.
 
@@ -232,6 +286,12 @@ class DataAdapter(ABC):
         kwargs['metric_list'] = self.metric_list
         return ReportGenerator.gen_report(subset_score_map, report_name, **kwargs)
 
+    def gen_prompt_data(self, prompt: str, **kwargs) -> dict:
+        if not isinstance(prompt, list):
+            prompt = [prompt]
+        prompt_data = PromptData(data=prompt, multi_choices=self.choices, system_prompt=self.system_prompt)
+        return prompt_data.to_dict()
+
     def gen_prompt(self, input_d: dict, subset_name: str, few_shot_list: list, **kwargs) -> Any:
         """
         Generate model prompt from raw input, unify the prompt format for different datasets.
@@ -295,3 +355,6 @@ class DataAdapter(ABC):
             The match result. Usually a score (float) for chat/multiple-choice-questions.
         """
         raise NotImplementedError
+
+    def llm_match(self, *args, **kwargs):
+        pass

evalscope/benchmarks/data_collection/data_collection_adapter.py

@@ -15,7 +15,6 @@ logger = get_logger()
 @Benchmark.register(
     name='data_collection',
     dataset_id='',  # dataset_id need to be set
-    model_adapter=ChatGenerationModelAdapter,
     subset_list=['default'],
     metric_list=['AverageAccuracy'],
     few_shot_num=0,

evalscope/benchmarks/general_mcq/general_mcq_adapter.py

@@ -3,9 +3,8 @@ import csv
 import os
 
 from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.constants import EvalType
+from evalscope.constants import EvalType, OutputType
 from evalscope.metrics.metrics import exact_match
-from evalscope.models import MultiChoiceModelAdapter
 from evalscope.utils import ResponseParser
 from evalscope.utils.logger import get_logger
 
@@ -16,22 +15,24 @@ logger = get_logger()
 
 @Benchmark.register(
     name='general_mcq',
+    pretty_name='General MCQ',
     dataset_id='general_mcq',
-    model_adapter=MultiChoiceModelAdapter,
+    model_adapter=OutputType.MULTIPLE_CHOICE,
+    output_types=[OutputType.MULTIPLE_CHOICE, OutputType.GENERATION],
     subset_list=['default'],
     metric_list=['AverageAccuracy'],
     few_shot_num=0,
     train_split='dev',
     eval_split='val',
     prompt_template='请回答问题，并选出其中的正确答案\n{query}',
-)
+    query_template='问题：{question}\n{choices}\n答案: {answer}\n\n')
 class GeneralMCQAdapter(DataAdapter):
 
-    choices = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J']
-
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
 
+        self.choices = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J']
+
     def load_from_disk(self, dataset_name_or_path, subset_list, work_dir, **kwargs) -> dict:
         data_dict = {}
         for subset_name in subset_list:
@@ -85,7 +86,7 @@ class GeneralMCQAdapter(DataAdapter):
 
         full_prompt = self.prompt_template.format(query=context)
 
-        return {'data': [full_prompt], 'multi_choices': self.choices, 'system_prompt': self.system_prompt}
+        return self.gen_prompt_data(full_prompt)
 
     def get_gold_answer(self, input_d: dict) -> str:
         # Get the gold choice
@@ -103,27 +104,19 @@ class GeneralMCQAdapter(DataAdapter):
         Returns:
             The parsed answer. Depending on the dataset. Usually a string for chat.
         """
-        if eval_type == EvalType.CHECKPOINT:
+        if self.model_adapter == OutputType.MULTIPLE_CHOICE:
             return result
-        elif eval_type == EvalType.SERVICE:
-            return ResponseParser.parse_first_option_with_choices(result, self.choices)
-        elif eval_type == EvalType.CUSTOM:
-            return ResponseParser.parse_first_option_with_choices(result, self.choices)
         else:
-            raise ValueError(f'Invalid eval_type: {eval_type}')
+            return ResponseParser.parse_first_option_with_choices(text=result, options=self.choices)
 
     def match(self, gold: str, pred: str) -> float:
         return exact_match(gold=gold, pred=pred)
 
-    @classmethod
-    def _format_example(cls, input_d: dict, include_answer=True):
-        example = '问题：' + input_d['question']
-        for choice in cls.choices:
-            if choice in input_d:
-                example += f'\n{choice}. {input_d[f"{choice}"]}'
+    def _format_example(self, input_d: dict, include_answer=True):
+        choices_str = '\n'.join([f'{choice}. {input_d[choice]}' for choice in self.choices if choice in input_d])
 
         if include_answer:
-            example += '\n答案: ' + input_d['answer'] + '\n\n'
+            return self.query_template.format(
+                question=input_d['question'], choices=choices_str, answer=input_d['answer'])
         else:
-            example += '\n答案: '
-        return example
+            return self.query_template.format(question=input_d['question'], choices=choices_str, answer='').rstrip()

evalscope/benchmarks/general_qa/general_qa_adapter.py

@@ -16,12 +16,12 @@ logger = get_logger()
 @Benchmark.register(
     name='general_qa',
     dataset_id='general_qa',
-    model_adapter=ChatGenerationModelAdapter,
     subset_list=['default'],
-    metric_list=['AverageBLEU'],
+    metric_list=['AverageBLEU', 'AverageRouge'],
     few_shot_num=0,
     train_split=None,
     eval_split='test',
+    prompt_template='请回答问题\n{query}',
 )
 class GeneralQAAdapter(DataAdapter):
     # TODO: set few_shot_num
@@ -30,18 +30,31 @@ class GeneralQAAdapter(DataAdapter):
 
         super().__init__(**kwargs)
 
-    def load(self, **kwargs) -> dict:
+    def load(self, dataset_name_or_path: str = None, subset_list: list = None, **kwargs) -> dict:
+        dataset_name_or_path = dataset_name_or_path or self.dataset_id
+        subset_list = subset_list or self.subset_list
 
-        data_file_list = glob.glob(os.path.join(self.dataset_id, '*.jsonl'))
+        data_file_dict = defaultdict(str)
         data_list = []
 
+        # get data file path and subset name
+        if os.path.isdir(dataset_name_or_path):
+            for subset_name in subset_list:
+                data_file_dict[subset_name] = os.path.join(dataset_name_or_path, f'{subset_name}.jsonl')
+        elif os.path.isfile(dataset_name_or_path):
+            cur_subset_name = os.path.basename(dataset_name_or_path).split('.')[0]
+            data_file_dict[cur_subset_name] = dataset_name_or_path
+        else:
+            raise ValueError(f'Invalid dataset path: {dataset_name_or_path}')
+
+        # load data from local disk
         try:
-            for file_path in data_file_list:
+            for subset_name, file_path in data_file_dict.items():
                 data_list.extend(jsonl_to_list(file_path))
         except Exception as e:
             raise ValueError(f'Failed to load data from {self.dataset_id}, got error: {e}')
 
-        data_dict = {'default': {'test': data_list}}
+        data_dict = {subset_name: {'test': data_list} for subset_name in data_file_dict.keys()}
 
         return data_dict
 
@@ -62,11 +75,9 @@ class GeneralQAAdapter(DataAdapter):
             logger.warning('The history is not included in the prompt for GeneralQA. \
                            To be supported in the future.')
 
-        prompt = input_d.get('question', '') or input_d.get('query', '')
-
-        # if len(history) > 0:
-        #     prompt = '\n'.join(history) + '\n' + prompt
-        return {'data': [prompt], 'system_prompt': self.system_prompt}
+        query = input_d.get('question', '') or input_d.get('query', '')
+        prompt = self.prompt_template.format(query=query)
+        return self.gen_prompt_data(prompt)
 
     def get_gold_answer(self, input_d: dict) -> str:
         """
@@ -101,13 +112,15 @@ class GeneralQAAdapter(DataAdapter):
 
         """
         res = dict()
-        rouge_dict = compute_rouge_score_one_sample_zh([pred], [gold])
-        bleu_dict = bleu_ngram_one_sample(pred, gold)
-        res.update(rouge_dict)
-        res.update(bleu_dict)
+        if 'AverageRouge' in self.metric_list:
+            rouge_dict = compute_rouge_score_one_sample_zh([pred], [gold])
+            res.update(rouge_dict)
+        if 'AverageBLEU' in self.metric_list:
+            bleu_dict = bleu_ngram_one_sample(pred, gold)
+            res.update(bleu_dict)
         return res
 
-    def compute_metric(self, review_res_list: List[dict]) -> List[dict]:
+    def compute_metric(self, review_res_list: List[dict], **kwargs) -> List[dict]:
         """
         compute weighted mean of the bleu score of all samples
 

evalscope/benchmarks/gpqa/gpqa_adapter.py

@@ -3,19 +3,20 @@ import random
 import re
 
 from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.constants import EvalType
+from evalscope.constants import EvalType, OutputType
 from evalscope.metrics import exact_match
-from evalscope.models import ChatGenerationModelAdapter
 
 
 @Benchmark.register(
     name='gpqa',
+    pretty_name='GPQA',
     dataset_id='modelscope/gpqa',
-    model_adapter=ChatGenerationModelAdapter,
+    model_adapter=OutputType.GENERATION,
+    output_types=[OutputType.MULTIPLE_CHOICE, OutputType.GENERATION],
     subset_list=['gpqa_extended', 'gpqa_main', 'gpqa_diamond'],
     metric_list=['AveragePass@1'],
     few_shot_num=5,
-    train_split='train',
+    train_split=None,
     eval_split='train',  # only have train split
     prompt_template='{query}\nPlease reason step by step, and put your final answer within \\boxed{{}}.',
 )
@@ -27,8 +28,9 @@ class GPQAAdapter(DataAdapter):
         self.choices = ['A', 'B', 'C', 'D']
         if self.few_shot_num and self.few_shot_num > 0:
             self.prompt_prefix = 'Here are some example questions from experts. Answer the final question yourself, following the format of the previous questions exactly.\n'  # noqa: E501
-            self.prompt_prefix += open(os.path.join(os.path.dirname(__file__), 'chain_of_thought.txt'),
-                                       'r').read() + '\nQuestion: '
+            self.prompt_prefix += open(
+                os.path.join(os.path.dirname(__file__), 'chain_of_thought.txt'), 'r',
+                encoding='utf-8').read() + '\nQuestion: '
         else:
             self.prompt_prefix = 'What is the correct answer to this question:'
 
@@ -50,7 +52,7 @@ class GPQAAdapter(DataAdapter):
         query = self.prompt_prefix + f"{input_d['Question']}\n{self.__form_options(processed_input_d['choices'])}"  # noqa: E501
 
         prompt = self.prompt_template.format(query=query)
-        return {'data': [prompt], 'multi_choices': self.choices, 'system_prompt': self.system_prompt}
+        return self.gen_prompt_data(prompt)
 
     def __process_input(self, input_d: dict) -> dict:
 
@@ -94,7 +96,10 @@ class GPQAAdapter(DataAdapter):
         """
         Parse the predicted result and extract proper answer.
         """
-        return GPQAAdapter.get_multiple_choice_answer(result)
+        if self.model_adapter == OutputType.MULTIPLE_CHOICE:
+            return result
+        else:
+            return GPQAAdapter.get_multiple_choice_answer(result)
 
     def match(self, gold: str, pred: str) -> float:
         """

evalscope/benchmarks/gsm8k/gsm8k_adapter.py

@@ -6,7 +6,6 @@ import os
 import re
 
 from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.models import ChatGenerationModelAdapter
 from evalscope.utils.io_utils import jsonl_to_list
 from evalscope.utils.logger import get_logger
 
@@ -15,12 +14,12 @@ logger = get_logger()
 
 @Benchmark.register(
     name='gsm8k',
+    pretty_name='GSM8K',
     dataset_id='modelscope/gsm8k',
-    model_adapter=ChatGenerationModelAdapter,
     subset_list=['main'],
     metric_list=['AverageAccuracy'],
     few_shot_num=4,
-    train_split='train',
+    train_split=None,
     eval_split='test',
     prompt_template="Question: {query}\nLet's think step by step\nAnswer:",
 )
@@ -76,7 +75,7 @@ class GSM8KAdapter(DataAdapter):
 
         full_prompt = context + self.prompt_template.format(query=input_d['question'])
 
-        return {'data': [full_prompt], 'system_prompt': self.system_prompt}
+        return self.gen_prompt_data(full_prompt)
 
     def get_gold_answer(self, input_d: dict) -> str:
         # Extract the gold answer from the input dict.

evalscope/benchmarks/hellaswag/hellaswag_adapter.py

@@ -4,9 +4,8 @@ import os
 import re
 
 from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.constants import EvalType
+from evalscope.constants import EvalType, OutputType
 from evalscope.metrics import exact_match
-from evalscope.models import ContinuationLogitsModelAdapter
 from evalscope.utils.io_utils import jsonl_to_list
 from evalscope.utils.logger import get_logger
 from evalscope.utils.utils import ResponseParser
@@ -18,8 +17,10 @@ logger = get_logger()
 
 @Benchmark.register(
     name='hellaswag',
+    pretty_name='HellaSwag',
     dataset_id='modelscope/hellaswag',
-    model_adapter=ContinuationLogitsModelAdapter,
+    model_adapter=OutputType.CONTINUOUS,
+    output_types=[OutputType.CONTINUOUS, OutputType.GENERATION],
     subset_list=['default'],
     metric_list=['AverageAccuracy'],
     few_shot_num=0,
@@ -30,8 +31,6 @@ logger = get_logger()
 )
 class HellaSwagAdapter(DataAdapter):
 
-    choices = ['0', '1', '2', '3']
-
     def __init__(self, **kwargs):
 
         few_shot_num = kwargs.get('few_shot_num', 0)
@@ -40,6 +39,7 @@ class HellaSwagAdapter(DataAdapter):
             kwargs['few_shot_num'] = 0
 
         super().__init__(**kwargs)
+        self.choices = ['0', '1', '2', '3']
 
     def load_from_disk(self, dataset_name_or_path, subset_list, work_dir, **kwargs) -> dict:
         data_dict = {}
@@ -89,7 +89,7 @@ class HellaSwagAdapter(DataAdapter):
 
         ctx_continuation_pair_list = [(context.strip(), ' ' + cont.strip()) for cont in endings]
 
-        return {'data': ctx_continuation_pair_list, 'multi_choices': self.choices, 'system_prompt': self.system_prompt}
+        return self.gen_prompt_data(ctx_continuation_pair_list)
 
     def get_gold_answer(self, input_d: dict) -> str:
         # Get the gold choice
@@ -107,7 +107,7 @@ class HellaSwagAdapter(DataAdapter):
         Returns:
             The parsed answer. Depending on the dataset. Usually a string for chat.
         """
-        if eval_type == EvalType.CHECKPOINT:
+        if self.model_adapter == OutputType.CONTINUOUS:
             # answer: in the form of [-2.3, -4.5, ...], len of self.choices
             result = np.array(result)
             endings: list = [self._preprocess(ending) for ending in raw_input_d['endings']]
@@ -115,12 +115,8 @@ class HellaSwagAdapter(DataAdapter):
             best_choice_idx = np.argmax(result / completion_len)
 
             return str(best_choice_idx)
-        elif eval_type == EvalType.SERVICE:
-            return ResponseParser.parse_first_option(result)
-        elif eval_type == EvalType.CUSTOM:
-            return ResponseParser.parse_first_option(result)
         else:
-            raise ValueError(f'Invalid eval_type: {eval_type}')
+            return ResponseParser.parse_first_option(result)
 
     def match(self, gold: str, pred: str) -> float:
         return exact_match(gold=str(gold), pred=str(pred))

evalscope/benchmarks/humaneval/humaneval_adapter.py

@@ -13,8 +13,8 @@ logger = get_logger()
 
 @Benchmark.register(
     name='humaneval',
+    pretty_name='HumanEval',
     dataset_id='modelscope/humaneval',
-    model_adapter=ChatGenerationModelAdapter,
     subset_list=['openai_humaneval'],
     metric_list=['Pass@1'],
     few_shot_num=0,
@@ -66,7 +66,7 @@ class HumanevalAdapter(DataAdapter):
         query = input_d['prompt']
         full_prompt = self.prompt_template.format(query=query)
 
-        return {'data': [full_prompt], 'system_prompt': self.system_prompt}
+        return self.gen_prompt_data(full_prompt)
 
     @classmethod
     def _postprocess(cls, text: str) -> str:

evalscope/benchmarks/ifeval/ifeval_adapter.py

@@ -5,13 +5,12 @@ from evalscope.benchmarks import Benchmark, DataAdapter
 from evalscope.benchmarks.ifeval.utils import process_results
 from evalscope.constants import EvalType
 from evalscope.metrics import Metric, mean, metric_registry
-from evalscope.models import ChatGenerationModelAdapter
 
 
 @Benchmark.register(
     name='ifeval',
+    pretty_name='IFEval',
     dataset_id='opencompass/ifeval',
-    model_adapter=ChatGenerationModelAdapter,
     subset_list=['default'],
     metric_list=[
         'prompt_level_strict_acc',
@@ -36,7 +35,7 @@ class IFEvalAdapter(DataAdapter):
         metric_registry.register(Metric(name='inst_level_loose_acc', object=mean))
 
     def gen_prompt(self, input_d: dict, subset_name: str, few_shot_list: list, **kwargs) -> Any:
-        return {'data': [input_d['prompt']], 'system_prompt': self.system_prompt}
+        return self.gen_prompt_data(input_d['prompt'])
 
     def get_gold_answer(self, input_d: dict) -> str:
         return input_d
@@ -47,7 +46,7 @@ class IFEvalAdapter(DataAdapter):
     def match(self, gold: Any, pred: Any) -> Dict:
         return process_results(gold, [pred])
 
-    def compute_metric(self, review_res_list: List[dict]) -> Any:
+    def compute_metric(self, review_res_list: List[dict], **kwargs) -> Any:
         # aggregate review results
         res_dict = defaultdict(list)
         for res in review_res_list:

evalscope/benchmarks/iquiz/iquiz_adapter.py

@@ -1,14 +1,15 @@
 from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.constants import EvalType
+from evalscope.constants import EvalType, OutputType
 from evalscope.metrics import exact_match
-from evalscope.models import ChatGenerationModelAdapter
 from evalscope.utils.utils import ResponseParser
 
 
 @Benchmark.register(
     name='iquiz',
+    pretty_name='IQuiz',
     dataset_id='AI-ModelScope/IQuiz',
-    model_adapter=ChatGenerationModelAdapter,
+    model_adapter=OutputType.GENERATION,
+    output_types=[OutputType.MULTIPLE_CHOICE, OutputType.GENERATION],
     subset_list=['IQ', 'EQ'],
     metric_list=['AverageAccuracy'],
     few_shot_num=0,
@@ -36,7 +37,7 @@ class IQuizAdapter(DataAdapter):
         """
         prompt = f"问题: {input_d['question']}\n"
         prompt += self.__form_options(input_d['choices'])
-        return {'data': [prompt], 'multi_choices': self.choices, 'system_prompt': self.system_prompt}
+        return self.gen_prompt_data(prompt)
 
     def __form_options(self, options: list):
         option_str = '选项:\n'
@@ -54,7 +55,10 @@ class IQuizAdapter(DataAdapter):
         """
         Parse the predicted result and extract proper answer.
         """
-        return ResponseParser.parse_first_option_with_choices(result, self.choices)
+        if self.model_adapter == OutputType.MULTIPLE_CHOICE:
+            return result
+        else:
+            return ResponseParser.parse_first_option_with_choices(result, self.choices)
 
     def match(self, gold: str, pred: str) -> float:
         """

evalscope/benchmarks/math_500/math_500_adapter.py

@@ -1,6 +1,5 @@
 from evalscope.benchmarks import Benchmark, DataAdapter
 from evalscope.metrics.math_parser import extract_answer, math_equal, strip_answer_string
-from evalscope.models import ChatGenerationModelAdapter
 from evalscope.utils.logger import get_logger
 
 # flake8: noqa
@@ -10,9 +9,9 @@ logger = get_logger()
 
 @Benchmark.register(
     name='math_500',
+    pretty_name='MATH-500',
     dataset_id='AI-ModelScope/MATH-500',
-    model_adapter=ChatGenerationModelAdapter,
-    subset_list=['default'],
+    subset_list=['Level 1', 'Level 2', 'Level 3', 'Level 4', 'Level 5'],
     metric_list=['AveragePass@1'],
     few_shot_num=0,
     train_split=None,
@@ -24,6 +23,12 @@ class Math500Adapter(DataAdapter):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
 
+    def load(self, **kwargs):
+        # default load all levels
+        kwargs['subset_list'] = ['default']
+        data_dict = super().load(**kwargs)
+        return self.reformat_subset(data_dict, subset_key='level', format='Level {}')
+
     def gen_prompt(self, input_d: dict, few_shot_list: list, **kwargs) -> dict:
         """
         Generate the prompt for the model input.
@@ -31,7 +36,7 @@ class Math500Adapter(DataAdapter):
         problem = input_d['problem']
         full_prompt = self.prompt_template.format(query=problem)
 
-        return {'data': [full_prompt], 'system_prompt': self.system_prompt}
+        return self.gen_prompt_data(full_prompt)
 
     def get_gold_answer(self, input_d: dict) -> str:
         # Extract the gold answer from the input dict.