evalscope 0.12.0__py3-none-any.whl → 0.12.1__py3-none-any.whl

evalscope/benchmarks/general_qa/general_qa_adapter.py

@@ -16,9 +16,8 @@ logger = get_logger()
 @Benchmark.register(
     name='general_qa',
     dataset_id='general_qa',
-    model_adapter=ChatGenerationModelAdapter,
     subset_list=['default'],
-    metric_list=['AverageBLEU'],
+    metric_list=['AverageBLEU', 'AverageRouge'],
     few_shot_num=0,
     train_split=None,
     eval_split='test',
@@ -31,18 +30,31 @@ class GeneralQAAdapter(DataAdapter):
 
         super().__init__(**kwargs)
 
-    def load(self, **kwargs) -> dict:
+    def load(self, dataset_name_or_path: str = None, subset_list: list = None, **kwargs) -> dict:
+        dataset_name_or_path = dataset_name_or_path or self.dataset_id
+        subset_list = subset_list or self.subset_list
 
-        data_file_list = glob.glob(os.path.join(self.dataset_id, '*.jsonl'))
+        data_file_dict = defaultdict(str)
         data_list = []
 
+        # get data file path and subset name
+        if os.path.isdir(dataset_name_or_path):
+            for subset_name in subset_list:
+                data_file_dict[subset_name] = os.path.join(dataset_name_or_path, f'{subset_name}.jsonl')
+        elif os.path.isfile(dataset_name_or_path):
+            cur_subset_name = os.path.basename(dataset_name_or_path).split('.')[0]
+            data_file_dict[cur_subset_name] = dataset_name_or_path
+        else:
+            raise ValueError(f'Invalid dataset path: {dataset_name_or_path}')
+
+        # load data from local disk
         try:
-            for file_path in data_file_list:
+            for subset_name, file_path in data_file_dict.items():
                 data_list.extend(jsonl_to_list(file_path))
         except Exception as e:
             raise ValueError(f'Failed to load data from {self.dataset_id}, got error: {e}')
 
-        data_dict = {'default': {'test': data_list}}
+        data_dict = {subset_name: {'test': data_list} for subset_name in data_file_dict.keys()}
 
         return data_dict
 
@@ -65,7 +77,7 @@ class GeneralQAAdapter(DataAdapter):
 
         query = input_d.get('question', '') or input_d.get('query', '')
         prompt = self.prompt_template.format(query=query)
-        return {'data': [prompt], 'system_prompt': self.system_prompt}
+        return self.gen_prompt_data(prompt)
 
     def get_gold_answer(self, input_d: dict) -> str:
         """
@@ -100,10 +112,12 @@ class GeneralQAAdapter(DataAdapter):
 
         """
         res = dict()
-        rouge_dict = compute_rouge_score_one_sample_zh([pred], [gold])
-        bleu_dict = bleu_ngram_one_sample(pred, gold)
-        res.update(rouge_dict)
-        res.update(bleu_dict)
+        if 'AverageRouge' in self.metric_list:
+            rouge_dict = compute_rouge_score_one_sample_zh([pred], [gold])
+            res.update(rouge_dict)
+        if 'AverageBLEU' in self.metric_list:
+            bleu_dict = bleu_ngram_one_sample(pred, gold)
+            res.update(bleu_dict)
         return res
 
     def compute_metric(self, review_res_list: List[dict], **kwargs) -> List[dict]:

evalscope/benchmarks/gpqa/gpqa_adapter.py

@@ -3,15 +3,16 @@ import random
 import re
 
 from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.constants import EvalType
+from evalscope.constants import EvalType, OutputType
 from evalscope.metrics import exact_match
-from evalscope.models import ChatGenerationModelAdapter
 
 
 @Benchmark.register(
     name='gpqa',
+    pretty_name='GPQA',
     dataset_id='modelscope/gpqa',
-    model_adapter=ChatGenerationModelAdapter,
+    model_adapter=OutputType.GENERATION,
+    output_types=[OutputType.MULTIPLE_CHOICE, OutputType.GENERATION],
     subset_list=['gpqa_extended', 'gpqa_main', 'gpqa_diamond'],
     metric_list=['AveragePass@1'],
     few_shot_num=5,
@@ -27,8 +28,9 @@ class GPQAAdapter(DataAdapter):
         self.choices = ['A', 'B', 'C', 'D']
         if self.few_shot_num and self.few_shot_num > 0:
             self.prompt_prefix = 'Here are some example questions from experts. Answer the final question yourself, following the format of the previous questions exactly.\n'  # noqa: E501
-            self.prompt_prefix += open(os.path.join(os.path.dirname(__file__), 'chain_of_thought.txt'),
-                                       'r').read() + '\nQuestion: '
+            self.prompt_prefix += open(
+                os.path.join(os.path.dirname(__file__), 'chain_of_thought.txt'), 'r',
+                encoding='utf-8').read() + '\nQuestion: '
         else:
             self.prompt_prefix = 'What is the correct answer to this question:'
 
@@ -50,7 +52,7 @@ class GPQAAdapter(DataAdapter):
         query = self.prompt_prefix + f"{input_d['Question']}\n{self.__form_options(processed_input_d['choices'])}"  # noqa: E501
 
         prompt = self.prompt_template.format(query=query)
-        return {'data': [prompt], 'multi_choices': self.choices, 'system_prompt': self.system_prompt}
+        return self.gen_prompt_data(prompt)
 
     def __process_input(self, input_d: dict) -> dict:
 
@@ -94,7 +96,10 @@ class GPQAAdapter(DataAdapter):
         """
         Parse the predicted result and extract proper answer.
         """
-        return GPQAAdapter.get_multiple_choice_answer(result)
+        if self.model_adapter == OutputType.MULTIPLE_CHOICE:
+            return result
+        else:
+            return GPQAAdapter.get_multiple_choice_answer(result)
 
     def match(self, gold: str, pred: str) -> float:
         """

evalscope/benchmarks/gsm8k/gsm8k_adapter.py

@@ -6,7 +6,6 @@ import os
 import re
 
 from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.models import ChatGenerationModelAdapter
 from evalscope.utils.io_utils import jsonl_to_list
 from evalscope.utils.logger import get_logger
 
@@ -15,8 +14,8 @@ logger = get_logger()
 
 @Benchmark.register(
     name='gsm8k',
+    pretty_name='GSM8K',
     dataset_id='modelscope/gsm8k',
-    model_adapter=ChatGenerationModelAdapter,
     subset_list=['main'],
     metric_list=['AverageAccuracy'],
     few_shot_num=4,
@@ -76,7 +75,7 @@ class GSM8KAdapter(DataAdapter):
 
         full_prompt = context + self.prompt_template.format(query=input_d['question'])
 
-        return {'data': [full_prompt], 'system_prompt': self.system_prompt}
+        return self.gen_prompt_data(full_prompt)
 
     def get_gold_answer(self, input_d: dict) -> str:
         # Extract the gold answer from the input dict.

evalscope/benchmarks/hellaswag/hellaswag_adapter.py

@@ -4,9 +4,8 @@ import os
 import re
 
 from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.constants import EvalType
+from evalscope.constants import EvalType, OutputType
 from evalscope.metrics import exact_match
-from evalscope.models import ContinuationLogitsModelAdapter
 from evalscope.utils.io_utils import jsonl_to_list
 from evalscope.utils.logger import get_logger
 from evalscope.utils.utils import ResponseParser
@@ -18,8 +17,10 @@ logger = get_logger()
 
 @Benchmark.register(
     name='hellaswag',
+    pretty_name='HellaSwag',
     dataset_id='modelscope/hellaswag',
-    model_adapter=ContinuationLogitsModelAdapter,
+    model_adapter=OutputType.CONTINUOUS,
+    output_types=[OutputType.CONTINUOUS, OutputType.GENERATION],
     subset_list=['default'],
     metric_list=['AverageAccuracy'],
     few_shot_num=0,
@@ -30,8 +31,6 @@ logger = get_logger()
 )
 class HellaSwagAdapter(DataAdapter):
 
-    choices = ['0', '1', '2', '3']
-
     def __init__(self, **kwargs):
 
         few_shot_num = kwargs.get('few_shot_num', 0)
@@ -40,6 +39,7 @@ class HellaSwagAdapter(DataAdapter):
             kwargs['few_shot_num'] = 0
 
         super().__init__(**kwargs)
+        self.choices = ['0', '1', '2', '3']
 
     def load_from_disk(self, dataset_name_or_path, subset_list, work_dir, **kwargs) -> dict:
         data_dict = {}
@@ -89,7 +89,7 @@ class HellaSwagAdapter(DataAdapter):
 
         ctx_continuation_pair_list = [(context.strip(), ' ' + cont.strip()) for cont in endings]
 
-        return {'data': ctx_continuation_pair_list, 'multi_choices': self.choices, 'system_prompt': self.system_prompt}
+        return self.gen_prompt_data(ctx_continuation_pair_list)
 
     def get_gold_answer(self, input_d: dict) -> str:
         # Get the gold choice
@@ -107,7 +107,7 @@ class HellaSwagAdapter(DataAdapter):
         Returns:
             The parsed answer. Depending on the dataset. Usually a string for chat.
         """
-        if eval_type == EvalType.CHECKPOINT:
+        if self.model_adapter == OutputType.CONTINUOUS:
             # answer: in the form of [-2.3, -4.5, ...], len of self.choices
             result = np.array(result)
             endings: list = [self._preprocess(ending) for ending in raw_input_d['endings']]
@@ -115,12 +115,8 @@ class HellaSwagAdapter(DataAdapter):
             best_choice_idx = np.argmax(result / completion_len)
 
             return str(best_choice_idx)
-        elif eval_type == EvalType.SERVICE:
-            return ResponseParser.parse_first_option(result)
-        elif eval_type == EvalType.CUSTOM:
-            return ResponseParser.parse_first_option(result)
         else:
-            raise ValueError(f'Invalid eval_type: {eval_type}')
+            return ResponseParser.parse_first_option(result)
 
     def match(self, gold: str, pred: str) -> float:
         return exact_match(gold=str(gold), pred=str(pred))

evalscope/benchmarks/humaneval/humaneval_adapter.py

@@ -13,8 +13,8 @@ logger = get_logger()
 
 @Benchmark.register(
     name='humaneval',
+    pretty_name='HumanEval',
     dataset_id='modelscope/humaneval',
-    model_adapter=ChatGenerationModelAdapter,
     subset_list=['openai_humaneval'],
     metric_list=['Pass@1'],
     few_shot_num=0,
@@ -66,7 +66,7 @@ class HumanevalAdapter(DataAdapter):
         query = input_d['prompt']
         full_prompt = self.prompt_template.format(query=query)
 
-        return {'data': [full_prompt], 'system_prompt': self.system_prompt}
+        return self.gen_prompt_data(full_prompt)
 
     @classmethod
     def _postprocess(cls, text: str) -> str:

evalscope/benchmarks/ifeval/ifeval_adapter.py

@@ -5,13 +5,12 @@ from evalscope.benchmarks import Benchmark, DataAdapter
 from evalscope.benchmarks.ifeval.utils import process_results
 from evalscope.constants import EvalType
 from evalscope.metrics import Metric, mean, metric_registry
-from evalscope.models import ChatGenerationModelAdapter
 
 
 @Benchmark.register(
     name='ifeval',
+    pretty_name='IFEval',
     dataset_id='opencompass/ifeval',
-    model_adapter=ChatGenerationModelAdapter,
     subset_list=['default'],
     metric_list=[
         'prompt_level_strict_acc',
@@ -36,7 +35,7 @@ class IFEvalAdapter(DataAdapter):
         metric_registry.register(Metric(name='inst_level_loose_acc', object=mean))
 
     def gen_prompt(self, input_d: dict, subset_name: str, few_shot_list: list, **kwargs) -> Any:
-        return {'data': [input_d['prompt']], 'system_prompt': self.system_prompt}
+        return self.gen_prompt_data(input_d['prompt'])
 
     def get_gold_answer(self, input_d: dict) -> str:
         return input_d

evalscope/benchmarks/iquiz/iquiz_adapter.py

@@ -1,14 +1,15 @@
 from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.constants import EvalType
+from evalscope.constants import EvalType, OutputType
 from evalscope.metrics import exact_match
-from evalscope.models import ChatGenerationModelAdapter
 from evalscope.utils.utils import ResponseParser
 
 
 @Benchmark.register(
     name='iquiz',
+    pretty_name='IQuiz',
     dataset_id='AI-ModelScope/IQuiz',
-    model_adapter=ChatGenerationModelAdapter,
+    model_adapter=OutputType.GENERATION,
+    output_types=[OutputType.MULTIPLE_CHOICE, OutputType.GENERATION],
     subset_list=['IQ', 'EQ'],
     metric_list=['AverageAccuracy'],
     few_shot_num=0,
@@ -36,7 +37,7 @@ class IQuizAdapter(DataAdapter):
         """
         prompt = f"问题: {input_d['question']}\n"
         prompt += self.__form_options(input_d['choices'])
-        return {'data': [prompt], 'multi_choices': self.choices, 'system_prompt': self.system_prompt}
+        return self.gen_prompt_data(prompt)
 
     def __form_options(self, options: list):
         option_str = '选项:\n'
@@ -54,7 +55,10 @@ class IQuizAdapter(DataAdapter):
         """
         Parse the predicted result and extract proper answer.
         """
-        return ResponseParser.parse_first_option_with_choices(result, self.choices)
+        if self.model_adapter == OutputType.MULTIPLE_CHOICE:
+            return result
+        else:
+            return ResponseParser.parse_first_option_with_choices(result, self.choices)
 
     def match(self, gold: str, pred: str) -> float:
         """

evalscope/benchmarks/math_500/math_500_adapter.py

@@ -1,9 +1,5 @@
-from collections import defaultdict
-
 from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.constants import AnswerKeys
 from evalscope.metrics.math_parser import extract_answer, math_equal, strip_answer_string
-from evalscope.models import ChatGenerationModelAdapter
 from evalscope.utils.logger import get_logger
 
 # flake8: noqa
@@ -13,8 +9,8 @@ logger = get_logger()
 
 @Benchmark.register(
     name='math_500',
+    pretty_name='MATH-500',
     dataset_id='AI-ModelScope/MATH-500',
-    model_adapter=ChatGenerationModelAdapter,
     subset_list=['Level 1', 'Level 2', 'Level 3', 'Level 4', 'Level 5'],
     metric_list=['AveragePass@1'],
     few_shot_num=0,
@@ -40,7 +36,7 @@ class Math500Adapter(DataAdapter):
         problem = input_d['problem']
         full_prompt = self.prompt_template.format(query=problem)
 
-        return {'data': [full_prompt], 'system_prompt': self.system_prompt}
+        return self.gen_prompt_data(full_prompt)
 
     def get_gold_answer(self, input_d: dict) -> str:
         # Extract the gold answer from the input dict.

evalscope/benchmarks/mmlu/mmlu_adapter.py

@@ -3,9 +3,8 @@ import csv
 import os
 
 from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.constants import EvalType
+from evalscope.constants import EvalType, OutputType
 from evalscope.metrics import exact_match
-from evalscope.models import MultiChoiceModelAdapter
 from evalscope.utils import ResponseParser
 from evalscope.utils.logger import get_logger
 
@@ -136,8 +135,10 @@ SUBJECT_MAPPING = {
 
 @Benchmark.register(
     name='mmlu',
+    pretty_name='MMLU',
     dataset_id='modelscope/mmlu',
-    model_adapter=MultiChoiceModelAdapter,
+    model_adapter=OutputType.MULTIPLE_CHOICE,
+    output_types=[OutputType.MULTIPLE_CHOICE, OutputType.GENERATION],
     subset_list=SUBSET_LIST,
     metric_list=['AverageAccuracy'],
     few_shot_num=5,
@@ -147,8 +148,6 @@ SUBJECT_MAPPING = {
 )
 class MMLUAdapter(DataAdapter):
 
-    choices = ['A', 'B', 'C', 'D']
-
     def __init__(self, **kwargs):
 
         few_shot_num = kwargs.get('few_shot_num', 5)
@@ -159,6 +158,7 @@ class MMLUAdapter(DataAdapter):
         super().__init__(**kwargs)
 
         self.category_map = {k: v[-1] for k, v in SUBJECT_MAPPING.items()}
+        self.choices = ['A', 'B', 'C', 'D']
 
     def load_from_disk(self, dataset_name_or_path, subset_list, work_dir, **kwargs) -> dict:
         data_dict = {}
@@ -227,7 +227,7 @@ class MMLUAdapter(DataAdapter):
 
         full_prompt = self.prompt_template.format(subset_name=self._format_subject(subset_name), query=query)
 
-        return {'data': [full_prompt], 'multi_choices': self.choices, 'system_prompt': self.system_prompt}
+        return self.gen_prompt_data(full_prompt)
 
     def get_gold_answer(self, input_d: dict) -> str:
         # Get the gold choice
@@ -245,26 +245,21 @@ class MMLUAdapter(DataAdapter):
         Returns:
             The parsed answer. Depending on the dataset. Usually a string for chat.
         """
-        if eval_type == EvalType.CHECKPOINT:
+        if self.model_adapter == OutputType.MULTIPLE_CHOICE:
             return result
-        elif eval_type == EvalType.SERVICE:
-            return ResponseParser.parse_first_option_with_choices(result, self.choices)
-        elif eval_type == EvalType.CUSTOM:
-            return ResponseParser.parse_first_option_with_choices(result, self.choices)
         else:
-            raise ValueError(f'Invalid eval_type: {eval_type}')
+            return ResponseParser.parse_first_option(result, self.choices)
 
     def match(self, gold: str, pred: str) -> float:
         return exact_match(gold=gold, pred=pred)
 
-    @classmethod
-    def _generate_prompt(cls, input_d: dict, include_answer=True) -> str:
+    def _generate_prompt(self, input_d: dict, include_answer=True) -> str:
 
         input_choices: list = [input_d['A'], input_d['B'], input_d['C'], input_d['D']]
 
         example: str = input_d['input']
-        for j in range(len(cls.choices)):
-            example += '\n{}. {}'.format(cls.choices[j], input_choices[j])
+        for j in range(len(self.choices)):
+            example += '\n{}. {}'.format(self.choices[j], input_choices[j])
 
         example += '\nAnswer:'
         if include_answer:

evalscope/benchmarks/mmlu_pro/mmlu_pro_adapter.py

@@ -2,9 +2,8 @@ from collections import defaultdict
 from typing import Any, Dict
 
 from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.constants import AnswerKeys, EvalType
+from evalscope.constants import EvalType, OutputType
 from evalscope.metrics import exact_match
-from evalscope.models import ChatGenerationModelAdapter
 from evalscope.utils.utils import ResponseParser
 
 SUBSET_LIST = [
@@ -15,8 +14,10 @@ SUBSET_LIST = [
 
 @Benchmark.register(
     name='mmlu_pro',
+    pretty_name='MMLU-Pro',
     dataset_id='modelscope/MMLU-Pro',
-    model_adapter=ChatGenerationModelAdapter,
+    model_adapter=OutputType.GENERATION,
+    output_types=[OutputType.MULTIPLE_CHOICE, OutputType.GENERATION],
     subset_list=SUBSET_LIST,
     metric_list=['AverageAccuracy'],
     few_shot_num=5,
@@ -47,7 +48,7 @@ class MMLUProAdapter(DataAdapter):
             self.__form_options(input_d['options']) + '\n'
 
         full_prompt = self.prompt_template.format(subset_name=subset_name, query=query)
-        return {'data': [full_prompt], 'system_prompt': self.system_prompt}
+        return self.gen_prompt_data(full_prompt)
 
     def format_fewshot_examples(self, few_shot_list):
         # load few-shot prompts for each category
@@ -88,7 +89,10 @@ class MMLUProAdapter(DataAdapter):
         Returns:
             The parsed answer. Depending on the dataset. Usually a string for chat.
         """
-        return ResponseParser.parse_first_option(result)
+        if self.model_adapter == OutputType.MULTIPLE_CHOICE:
+            return result
+        else:
+            return ResponseParser.parse_first_option(result)
 
     def match(self, gold: str, pred: str) -> float:
         """

evalscope/benchmarks/musr/musr_adapter.py

@@ -2,9 +2,8 @@ import ast
 from typing import Any
 
 from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.constants import EvalType
+from evalscope.constants import EvalType, OutputType
 from evalscope.metrics import exact_match
-from evalscope.models import ChatGenerationModelAdapter
 from evalscope.utils.utils import ResponseParser
 
 
@@ -12,7 +11,8 @@ from evalscope.utils.utils import ResponseParser
     name='musr',
     pretty_name='MuSR',
     dataset_id='AI-ModelScope/MuSR',
-    model_adapter=ChatGenerationModelAdapter,
+    model_adapter=OutputType.GENERATION,
+    output_types=[OutputType.MULTIPLE_CHOICE, OutputType.GENERATION],
     subset_list=['murder_mysteries', 'object_placements', 'team_allocation'],
     metric_list=['AverageAccuracy'],
     few_shot_num=0,
@@ -41,7 +41,7 @@ class MuSRAdapter(DataAdapter):
         full_prompt = self.prompt_template.format(
             narrative=input_d['narrative'], question=input_d['question'], choices=choices)
 
-        return {'data': [full_prompt], 'system_prompt': self.system_prompt}
+        return self.gen_prompt_data(full_prompt)
 
     def format_choice(self, options: list):
         option_str = ''
@@ -59,7 +59,10 @@ class MuSRAdapter(DataAdapter):
         """
         Parse the predicted result and extract proper answer.
         """
-        return ResponseParser.parse_first_option(result)
+        if self.model_adapter == OutputType.MULTIPLE_CHOICE:
+            return result
+        else:
+            return ResponseParser.parse_first_option(result)
 
     def match(self, gold: str, pred: str) -> float:
         """

evalscope/benchmarks/process_bench/process_bench_adapter.py

@@ -5,7 +5,6 @@ from typing import Any, List
 from evalscope.benchmarks import Benchmark, DataAdapter
 from evalscope.constants import AnswerKeys, EvalType
 from evalscope.metrics import Metric, mean, metric_registry, simple_f1_score
-from evalscope.models import ChatGenerationModelAdapter
 
 cur_path = os.path.dirname(os.path.abspath(__file__))
 
@@ -14,7 +13,6 @@ cur_path = os.path.dirname(os.path.abspath(__file__))
     name='process_bench',
     pretty_name='ProcessBench',
     dataset_id='Qwen/ProcessBench',
-    model_adapter=ChatGenerationModelAdapter,
     subset_list=['gsm8k', 'math', 'olympiadbench', 'omnimath'],
     metric_list=['error_acc', 'correct_acc', 'simple_f1_score'],
     few_shot_num=0,
@@ -26,7 +24,7 @@ class ProcessBenchAdapter(DataAdapter):
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
 
-        self.prompt_template = open(os.path.join(cur_path, 'critique_template.txt')).read()
+        self.prompt_template = open(os.path.join(cur_path, 'critique_template.txt'), encoding='utf-8').read()
 
         # register metrics
         metric_registry.register(Metric(name='error_acc', object=mean))
@@ -50,7 +48,7 @@ class ProcessBenchAdapter(DataAdapter):
 
         full_prompt = self.prompt_template.format(problem=problem, tagged_response=tagged_response)
 
-        return {'data': [full_prompt], 'system_prompt': self.system_prompt}
+        return self.gen_prompt_data(full_prompt)
 
     def get_gold_answer(self, input_d: dict) -> str:
         """
@@ -84,7 +82,12 @@ class ProcessBenchAdapter(DataAdapter):
                 correct_data.append(res)
             else:
                 error_data.append(res)
-        data = {'error_acc': error_data, 'correct_acc': correct_data, 'simple_f1_score': (correct_data, error_data)}
+        data = {}
+        if len(correct_data) != 0:
+            data.update({'correct_acc': correct_data})
+        if len(error_data) != 0:
+            data.update({'error_acc': error_data})
+        data.update({'simple_f1_score': (correct_data, error_data)})
         return super().compute_metric(data)
 
     @staticmethod

evalscope/benchmarks/race/race_adapter.py

@@ -3,9 +3,8 @@
 import os
 
 from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.constants import EvalType
+from evalscope.constants import EvalType, OutputType
 from evalscope.metrics import exact_match
-from evalscope.models import MultiChoiceModelAdapter
 from evalscope.utils import ResponseParser
 from evalscope.utils.io_utils import jsonl_to_list
 from evalscope.utils.logger import get_logger
@@ -17,8 +16,10 @@ logger = get_logger()
 
 @Benchmark.register(
     name='race',
+    pretty_name='RACE',
     dataset_id='modelscope/race',
-    model_adapter=MultiChoiceModelAdapter,
+    model_adapter=OutputType.MULTIPLE_CHOICE,
+    output_types=[OutputType.MULTIPLE_CHOICE, OutputType.GENERATION],
     subset_list=['high', 'middle'],
     metric_list=['AverageAccuracy'],
     few_shot_num=3,
@@ -27,8 +28,6 @@ logger = get_logger()
 )
 class RACEAdapter(DataAdapter):
 
-    choices = ['A', 'B', 'C', 'D']
-
     def __init__(self, **kwargs):
         few_shot_num = kwargs.get('few_shot_num', 3)
         if few_shot_num > 3:
@@ -37,6 +36,8 @@ class RACEAdapter(DataAdapter):
 
         super().__init__(**kwargs)
 
+        self.choices = ['A', 'B', 'C', 'D']
+
     def load_from_disk(self, dataset_name_or_path, subset_list, work_dir, **kwargs) -> dict:
         data_dict = {}
         for subset_name in subset_list:
@@ -82,7 +83,7 @@ class RACEAdapter(DataAdapter):
 
         full_prompt: str = context.strip() + self._generate_prompt(input_d=input_d, include_answer=False)
 
-        return {'data': [full_prompt], 'multi_choices': self.choices, 'system_prompt': self.system_prompt}
+        return self.gen_prompt_data(full_prompt)
 
     def get_gold_answer(self, input_d: dict) -> str:
         # Get the gold choice
@@ -100,26 +101,21 @@ class RACEAdapter(DataAdapter):
         Returns:
             The parsed answer. Depending on the dataset. Usually a string for chat.
         """
-        if eval_type == EvalType.CHECKPOINT:
+        if self.model_adapter == OutputType.MULTIPLE_CHOICE:
             return result
-        elif eval_type == EvalType.SERVICE:
-            return ResponseParser.parse_first_option_with_choices(result, self.choices)  # TODO: to be checked !
-        elif eval_type == EvalType.CUSTOM:
-            return ResponseParser.parse_first_option_with_choices(result, self.choices)  # TODO: to be checked !
         else:
-            raise ValueError(f'Unknown eval_type: {eval_type}')
+            return ResponseParser.parse_first_option_with_choices(result, self.choices)
 
     def match(self, gold: str, pred: str) -> float:
         return exact_match(gold=gold, pred=pred)
 
-    @classmethod
-    def _generate_prompt(cls, input_d: dict, include_answer=True) -> str:
+    def _generate_prompt(self, input_d: dict, include_answer=True) -> str:
 
         input_choices: list = input_d['options']
 
         example: str = 'Article:\n{}\nQuestion:\n{}'.format(input_d['article'], input_d['question'])
-        for j in range(len(cls.choices)):
-            example += '\n{}. {}'.format(cls.choices[j], input_choices[j])
+        for j in range(len(self.choices)):
+            example += '\n{}. {}'.format(self.choices[j], input_choices[j])
 
         example += '\nAnswer:'
         if include_answer:

evalscope/benchmarks/simple_qa/simple_qa_adapter.py

@@ -0,0 +1,20 @@
+from evalscope.benchmarks import Benchmark, DataAdapter
+from evalscope.utils.logger import get_logger
+
+# flake8: noqa
+
+logger = get_logger()
+
+
+@Benchmark.register(
+    name='simple_qa',
+    pretty_name='SimpleQA',
+    dataset_id='AI-ModelScope/SimpleQA',
+    metric_list=['AverageAccuracy'],
+    few_shot_num=0,
+    train_split=None,
+    eval_split='test')
+class SimpleQAAdapter(DataAdapter):
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)