evalscope 0.12.0__py3-none-any.whl → 0.12.1__py3-none-any.whl

evalscope/arguments.py

@@ -1,7 +1,7 @@
 import argparse
 import json
 
-from evalscope.constants import EvalBackend, EvalStage, EvalType
+from evalscope.constants import EvalBackend, EvalStage, EvalType, OutputType
 
 
 class ParseStrArgsAction(argparse.Action):

evalscope/benchmarks/aime/aime24_adapter.py

@@ -1,6 +1,6 @@
 from evalscope.benchmarks import Benchmark, DataAdapter
+from evalscope.constants import OutputType
 from evalscope.metrics.math_parser import extract_answer, math_equal, strip_answer_string
-from evalscope.models import ChatGenerationModelAdapter
 from evalscope.utils.logger import get_logger
 
 # flake8: noqa
@@ -10,8 +10,8 @@ logger = get_logger()
 
 @Benchmark.register(
     name='aime24',
+    pretty_name='AIME-2024',
     dataset_id='HuggingFaceH4/aime_2024',
-    model_adapter=ChatGenerationModelAdapter,
     subset_list=['default'],
     metric_list=['AveragePass@1'],
     few_shot_num=0,
@@ -31,7 +31,7 @@ class AIME24Adapter(DataAdapter):
         problem = input_d['problem']
         full_prompt = self.prompt_template.format(query=problem)
 
-        return {'data': [full_prompt], 'system_prompt': self.system_prompt}
+        return self.gen_prompt_data(full_prompt)
 
     def get_gold_answer(self, input_d: dict) -> str:
         # Extract the gold answer from the input dict.

evalscope/benchmarks/aime/aime25_adapter.py

@@ -1,6 +1,6 @@
 from evalscope.benchmarks import Benchmark, DataAdapter
+from evalscope.constants import OutputType
 from evalscope.metrics.math_parser import extract_answer, math_equal, strip_answer_string
-from evalscope.models import ChatGenerationModelAdapter
 from evalscope.utils.logger import get_logger
 
 # flake8: noqa
@@ -10,8 +10,8 @@ logger = get_logger()
 
 @Benchmark.register(
     name='aime25',
+    pretty_name='AIME-2025',
     dataset_id='TIGER-Lab/AIME25',
-    model_adapter=ChatGenerationModelAdapter,
     subset_list=['default'],
     metric_list=['AveragePass@1'],
     few_shot_num=0,
@@ -31,7 +31,7 @@ class AIME25Adapter(DataAdapter):
         problem = input_d['question']
         full_prompt = self.prompt_template.format(query=problem)
 
-        return {'data': [full_prompt], 'system_prompt': self.system_prompt}
+        return self.gen_prompt_data(full_prompt)
 
     def get_gold_answer(self, input_d: dict) -> str:
         # Extract the gold answer from the input dict.

evalscope/benchmarks/arc/arc_adapter.py

@@ -4,9 +4,8 @@ import json
 import os
 
 from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.constants import EvalType
+from evalscope.constants import EvalType, OutputType
 from evalscope.metrics import exact_match
-from evalscope.models import MultiChoiceModelAdapter
 from evalscope.utils import ResponseParser
 from evalscope.utils.logger import get_logger
 
@@ -17,19 +16,20 @@ logger = get_logger()
 
 @Benchmark.register(
     name='arc',
+    pretty_name='ARC',
     dataset_id='modelscope/ai2_arc',
-    model_adapter=MultiChoiceModelAdapter,
+    model_adapter=OutputType.MULTIPLE_CHOICE,
+    output_types=[OutputType.MULTIPLE_CHOICE, OutputType.GENERATION],
     subset_list=['ARC-Easy', 'ARC-Challenge'],
     metric_list=['AverageAccuracy'],
     few_shot_num=0,
     train_split='train',
     eval_split='test',
-    prompt_template='',
+    prompt_template=
+    'The following are multiple choice questions, please output correct answer in the form of A or B or C or D, do not output explanation:\n{query}',
 )
 class ARCAdapter(DataAdapter):
 
-    choices = ['A', 'B', 'C', 'D']
-
     def __init__(self, **kwargs):
         few_shot_num = kwargs.get('few_shot_num', None)
         if few_shot_num is None:
@@ -42,6 +42,8 @@ class ARCAdapter(DataAdapter):
 
         super().__init__(**kwargs)
 
+        self.choices = ['A', 'B', 'C', 'D']
+
     def load_from_disk(self, dataset_name_or_path, subset_list, work_dir, **kwargs) -> dict:
         """
         Load the dataset from local disk.
@@ -60,7 +62,7 @@ class ARCAdapter(DataAdapter):
             for split_name in ['Train', 'Test']:
                 split_path = os.path.join(subset_path, f'{subset_name}-{split_name}.jsonl')
                 if os.path.exists(split_path):
-                    with open(split_path, 'r', errors='ignore') as in_f:
+                    with open(split_path, 'r', errors='ignore', encoding='utf-8') as in_f:
                         rows = []
                         for line in in_f:
                             item = json.loads(line.strip())
@@ -107,12 +109,11 @@ class ARCAdapter(DataAdapter):
             {'data': ['xxx'], 'multi_choices': ['A', 'B', 'C', 'D']}
         """
         few_shot_prompts = [self._generate_prompt(input_d=sample, include_answer=True) for sample in few_shot_list]
-        context: str = '\n'.join(few_shot_prompts)
+        context = '\n'.join(few_shot_prompts) + self._generate_prompt(input_d=input_d, include_answer=False)
 
-        # context = f'The following are multiple choice questions, please output correct answer in the form of A or B or C or D, do not output explanation:\n {context}'
-        full_prompt: str = context + self._generate_prompt(input_d=input_d, include_answer=False)
+        full_prompt = self.prompt_template.format(query=context)
 
-        return {'data': [full_prompt], 'multi_choices': self.choices, 'system_prompt': self.system_prompt}
+        return self.gen_prompt_data(full_prompt)
 
     def get_gold_answer(self, input_d: dict) -> str:
         # Get the gold choice
@@ -130,14 +131,10 @@ class ARCAdapter(DataAdapter):
         Returns:
             The parsed answer. Depending on the dataset. Usually a string for chat.
         """
-        if eval_type == EvalType.CHECKPOINT:
+        if self.model_adapter == OutputType.MULTIPLE_CHOICE:
             return result
-        elif eval_type == EvalType.SERVICE:
-            return ResponseParser.parse_first_option_with_choices(text=result, options=self.choices)
-        elif eval_type == EvalType.CUSTOM:
-            return ResponseParser.parse_first_option_with_choices(text=result, options=self.choices)
         else:
-            raise ValueError(f'Invalid eval_type: {eval_type}')
+            return ResponseParser.parse_first_capital(text=result, options=self.choices)
 
     def match(self, gold: str, pred: str) -> float:
         return exact_match(gold=gold, pred=pred)

evalscope/benchmarks/bbh/bbh_adapter.py

@@ -8,8 +8,6 @@ import re
 from evalscope.benchmarks import Benchmark, DataAdapter
 from evalscope.constants import AnswerKeys
 from evalscope.metrics import exact_match
-from evalscope.models.chat_adapter import ChatGenerationModelAdapter
-from evalscope.utils import ResponseParser
 from evalscope.utils.logger import get_logger
 
 # flake8: noqa
@@ -60,8 +58,8 @@ SUBSET_LIST = MULTIPLE_CHOICE_LIST + FREE_FORM_LIST
 
 @Benchmark.register(
     name='bbh',
+    pretty_name='BBH',
     dataset_id='modelscope/bbh',
-    model_adapter=ChatGenerationModelAdapter,
     subset_list=SUBSET_LIST,
     metric_list=['AverageAccuracy'],
     few_shot_num=3,
@@ -94,7 +92,7 @@ class BBHAdapter(DataAdapter):
                 else:
                     file_path: str = os.path.join(work_dir, dataset_name_or_path, f'{subset_name}.json')
                 if os.path.exists(file_path):
-                    with open(file_path, 'r') as f:
+                    with open(file_path, 'r', encoding='utf-8') as f:
                         examples = json.load(f)['examples']
                         if subset_name in data_dict:
                             data_dict[subset_name].update({split_name: examples})
@@ -125,7 +123,7 @@ class BBHAdapter(DataAdapter):
             cot_prompts = ''
         full_prompt = cot_prompts + self.prompt_template.format(query=input_d['input'])
 
-        return {'data': [full_prompt], 'system_prompt': self.system_prompt}
+        return self.gen_prompt_data(full_prompt)
 
     def gen_prompts(self, data_dict: dict) -> dict:
         """
@@ -153,7 +151,9 @@ class BBHAdapter(DataAdapter):
         for sub_name, sub_data_dict in data_dict.items():
             few_shot_data = []
             if self.few_shot_num > 0:
-                with open(os.path.join(os.path.dirname(__file__), 'cot_prompts', f'{sub_name}.txt'), 'r') as f:
+                with open(
+                        os.path.join(os.path.dirname(__file__), 'cot_prompts', f'{sub_name}.txt'), 'r',
+                        encoding='utf-8') as f:
                     cot_prompt_str = f.read()
                 few_shot_data = [cot_prompt_str]
 

evalscope/benchmarks/benchmark.py

@@ -1,12 +1,13 @@
 import copy
+from collections import OrderedDict
 from dataclasses import dataclass, field
 from typing import TYPE_CHECKING, Dict, List, Optional
 
+from evalscope.constants import OutputType
+
 if TYPE_CHECKING:
     from evalscope.benchmarks import DataAdapter
 
-from evalscope.models import BaseModelAdapter
-
 BENCHMARK_MAPPINGS = {}
 
 
@@ -15,8 +16,9 @@ class BenchmarkMeta:
     name: str
     dataset_id: str
     data_adapter: 'DataAdapter'
-    model_adapter: BaseModelAdapter
-    subset_list: List[str] = field(default_factory=list)
+    model_adapter: Optional[str] = OutputType.GENERATION
+    output_types: Optional[List[str]] = field(default_factory=lambda: [OutputType.GENERATION])
+    subset_list: List[str] = field(default_factory=lambda: ['default'])
     metric_list: List[str] = field(default_factory=list)
     few_shot_num: int = 0
     few_shot_random: bool = False
@@ -26,6 +28,7 @@ class BenchmarkMeta:
     system_prompt: Optional[str] = None
     query_template: Optional[str] = None
     pretty_name: Optional[str] = None
+    filters: Optional[OrderedDict] = None
 
     def _update(self, args: dict):
         if args.get('local_path'):
@@ -39,10 +42,7 @@ class BenchmarkMeta:
     def to_string_dict(self) -> dict:
         cur_dict = copy.deepcopy(self.__dict__)
         # cur_dict['data_adapter'] = self.data_adapter.__name__
-        # cur_dict['model_adapter'] = self.model_adapter.__name__
-        # cur_dict['metric_list'] = [metric['name'] for metric in self.metric_list]
         del cur_dict['data_adapter']
-        del cur_dict['model_adapter']
         return cur_dict
 
     def get_data_adapter(self, config: dict = {}) -> 'DataAdapter':
@@ -66,13 +66,13 @@ class Benchmark:
         return benchmark
 
     @classmethod
-    def register(cls, name: str, dataset_id: str, model_adapter: BaseModelAdapter, **kwargs):
+    def register(cls, name: str, dataset_id: str, **kwargs):
 
         def register_wrapper(data_adapter):
             if name in BENCHMARK_MAPPINGS:
                 raise Exception(f'Benchmark {name} already registered')
             BENCHMARK_MAPPINGS[name] = BenchmarkMeta(
-                name=name, data_adapter=data_adapter, model_adapter=model_adapter, dataset_id=dataset_id, **kwargs)
+                name=name, data_adapter=data_adapter, dataset_id=dataset_id, **kwargs)
             return data_adapter
 
         return register_wrapper

evalscope/benchmarks/ceval/ceval_adapter.py

@@ -3,9 +3,8 @@ import csv
 import os
 
 from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.constants import EvalType
+from evalscope.constants import EvalType, OutputType
 from evalscope.metrics.metrics import exact_match
-from evalscope.models import MultiChoiceModelAdapter
 from evalscope.utils import ResponseParser
 from evalscope.utils.logger import get_logger
 
@@ -126,8 +125,10 @@ SUBJECT_MAPPING = {
 
 @Benchmark.register(
     name='ceval',
+    pretty_name='C-Eval',
     dataset_id='modelscope/ceval-exam',
-    model_adapter=MultiChoiceModelAdapter,
+    model_adapter=OutputType.MULTIPLE_CHOICE,
+    output_types=[OutputType.MULTIPLE_CHOICE, OutputType.GENERATION],
     subset_list=SUBSET_LIST,
     metric_list=['AverageAccuracy'],
     few_shot_num=0,
@@ -137,8 +138,6 @@ SUBJECT_MAPPING = {
 )
 class CEVALAdapter(DataAdapter):
 
-    choices = ['A', 'B', 'C', 'D']
-
     def __init__(self, **kwargs):
 
         few_shot_num = kwargs.get('few_shot_num', 0)
@@ -148,6 +147,7 @@ class CEVALAdapter(DataAdapter):
         super().__init__(**kwargs)
 
         self.category_map = {k: v[-1] for k, v in SUBJECT_MAPPING.items()}
+        self.choices = ['A', 'B', 'C', 'D']
 
     def load_from_disk(self, dataset_name_or_path, subset_list, work_dir, **kwargs) -> dict:
         data_dict = {}
@@ -207,7 +207,7 @@ class CEVALAdapter(DataAdapter):
         subject_name: str = SUBJECT_MAPPING.get(subset_name)[1] if SUBJECT_MAPPING.get(subset_name) else subset_name
         full_prompt = self.prompt_template.format(subset_name=subject_name, query=query)
 
-        return {'data': [full_prompt], 'multi_choices': self.choices, 'system_prompt': self.system_prompt}
+        return self.gen_prompt_data(full_prompt)
 
     def get_gold_answer(self, input_d: dict) -> str:
         # Get the gold choice
@@ -225,22 +225,17 @@ class CEVALAdapter(DataAdapter):
         Returns:
             The parsed answer. Depending on the dataset. Usually a string for chat.
         """
-        if eval_type == EvalType.CHECKPOINT:
+        if self.model_adapter == OutputType.MULTIPLE_CHOICE:
             return result
-        elif eval_type == EvalType.SERVICE:
-            return ResponseParser.parse_first_option_with_choices(result, self.choices)
-        elif eval_type == EvalType.CUSTOM:
-            return ResponseParser.parse_first_option_with_choices(result, self.choices)
         else:
-            raise ValueError(f'Invalid eval_type: {eval_type}')
+            return ResponseParser.parse_first_option_with_choices(text=result, options=self.choices)
 
     def match(self, gold: str, pred: str) -> float:
         return exact_match(gold=gold, pred=pred)
 
-    @classmethod
-    def _format_example(cls, input_d: dict, include_answer=True):
+    def _format_example(self, input_d: dict, include_answer=True):
         example = '问题：' + input_d['question']
-        for choice in cls.choices:
+        for choice in self.choices:
             example += f'\n{choice}. {input_d[f"{choice}"]}'
 
         if include_answer:

evalscope/benchmarks/cmmlu/cmmlu_adapter.py

@@ -4,9 +4,8 @@ import csv
 import os
 
 from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.constants import EvalType
+from evalscope.constants import EvalType, OutputType
 from evalscope.metrics import exact_match
-from evalscope.models import MultiChoiceModelAdapter
 from evalscope.utils import ResponseParser
 from evalscope.utils.logger import get_logger
 
@@ -103,8 +102,10 @@ SUBJECT_MAPPING = {
 
 @Benchmark.register(
     name='cmmlu',
+    pretty_name='C-MMLU',
     dataset_id='modelscope/cmmlu',
-    model_adapter=MultiChoiceModelAdapter,
+    model_adapter=OutputType.MULTIPLE_CHOICE,
+    output_types=[OutputType.MULTIPLE_CHOICE, OutputType.GENERATION],
     subset_list=SUBSET_LIST,
     metric_list=['AverageAccuracy'],
     few_shot_num=5,
@@ -114,12 +115,11 @@ SUBJECT_MAPPING = {
 )
 class CMMLUAdapter(DataAdapter):
 
-    choices = ['A', 'B', 'C', 'D']
-
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
 
         self.category_map = {k: v[-1] for k, v in SUBJECT_MAPPING.items()}
+        self.choices = ['A', 'B', 'C', 'D']
 
     def load_from_disk(self, dataset_name_or_path, subset_list, work_dir, **kwargs) -> dict:
         data_dict = {}
@@ -172,7 +172,7 @@ class CMMLUAdapter(DataAdapter):
 
         full_prompt = self.prompt_template.format(subset_name=self._format_subject(subset_name), query=context.strip())
 
-        return {'data': [full_prompt], 'multi_choices': self.choices, 'system_prompt': self.system_prompt}
+        return self.gen_prompt_data(full_prompt)
 
     def get_gold_answer(self, input_d: dict) -> str:
         # Get the gold choice
@@ -190,26 +190,21 @@ class CMMLUAdapter(DataAdapter):
         Returns:
             The parsed answer. Depending on the dataset. Usually a string for chat.
         """
-        if eval_type == EvalType.CHECKPOINT:
+        if self.model_adapter == OutputType.MULTIPLE_CHOICE:
             return result
-        elif eval_type == EvalType.SERVICE:
-            return ResponseParser.parse_first_option_with_choices(result, self.choices)
-        elif eval_type == EvalType.CUSTOM:
-            return ResponseParser.parse_first_option_with_choices(result, self.choices)
         else:
-            raise ValueError(f'Invalid eval_type: {eval_type}')
+            return ResponseParser.parse_first_option_with_choices(text=result, options=self.choices)
 
     def match(self, gold: str, pred: str) -> float:
         return exact_match(gold=gold, pred=pred)
 
-    @classmethod
-    def _generate_prompt(cls, input_d: dict, include_answer=True) -> str:
+    def _generate_prompt(self, input_d: dict, include_answer=True) -> str:
 
         input_choices: list = [input_d['A'], input_d['B'], input_d['C'], input_d['D']]
 
         example: str = input_d['Question']
-        for j in range(len(cls.choices)):
-            example += '\n{}. {}'.format(cls.choices[j], input_choices[j])
+        for j in range(len(self.choices)):
+            example += '\n{}. {}'.format(self.choices[j], input_choices[j])
 
         example += '\nAnswer:'
         if include_answer:

evalscope/benchmarks/competition_math/competition_math_adapter.py

@@ -18,8 +18,8 @@ logger = get_logger()
 
 @Benchmark.register(
     name='competition_math',
+    pretty_name='MATH',
     dataset_id='modelscope/competition_math',
-    model_adapter=ChatGenerationModelAdapter,
     subset_list=['Level 1', 'Level 2', 'Level 3', 'Level 4', 'Level 5'],
     metric_list=['AveragePass@1'],
     few_shot_num=4,
@@ -58,7 +58,7 @@ class CompetitionMathAdapter(DataAdapter):
                 split_data = []
                 for file_path in split_files:
                     if os.path.exists(file_path):
-                        with open(file_path, 'r') as f:
+                        with open(file_path, 'r', encoding='utf-8') as f:
                             split_data.append(json.load(f))
                 data_dict[subset_name][split_name] = split_data
 
@@ -81,7 +81,7 @@ class CompetitionMathAdapter(DataAdapter):
         use_fewshot = self.few_shot_num > 0
         query = self._generate_prompt(input_d, use_fewshot=use_fewshot)
         full_prompt = self.prompt_template.format(query=query)
-        return {'data': [full_prompt], 'system_prompt': self.system_prompt}
+        return self.gen_prompt_data(full_prompt)
 
     def get_gold_answer(self, input_d: dict) -> str:
         # Extract the gold answer from the input dict.

evalscope/benchmarks/data_adapter.py

@@ -5,6 +5,7 @@ from abc import ABC, abstractmethod
 from collections import defaultdict
 from typing import Any, List, Optional, Union
 
+from evalscope.benchmarks.utils import PromptData, preprocess_decorator
 from evalscope.constants import DEFAULT_DATASET_CACHE_DIR, AnswerKeys, EvalType, HubType
 from evalscope.metrics.named_metrics import metric_registry
 from evalscope.report import Report, ReportGenerator
@@ -18,6 +19,7 @@ class DataAdapter(ABC):
     def __init__(self,
                  name: str,
                  dataset_id: str,
+                 model_adapter: str,
                  subset_list: list,
                  metric_list: List[str],
                  few_shot_num: Optional[int] = 0,
@@ -48,6 +50,7 @@ class DataAdapter(ABC):
         """
         self.name = name
         self.dataset_id = dataset_id
+        self.model_adapter = model_adapter
         self.subset_list = subset_list
         self.metric_list = metric_list
         self.few_shot_num = few_shot_num
@@ -59,6 +62,15 @@ class DataAdapter(ABC):
         self.pretty_name = pretty_name
         self.config_kwargs = kwargs
         self.category_map = kwargs.get('category_map', {})
+        self.choices = kwargs.get('choices', None)
+
+    def __init_subclass__(cls, **kwargs):
+        super().__init_subclass__(**kwargs)
+
+        # find and decorate parse_pred_result method
+        if hasattr(cls, 'parse_pred_result'):
+            original_method = cls.parse_pred_result
+            cls.parse_pred_result = preprocess_decorator(original_method)
 
     def load(self,
              dataset_name_or_path: str = None,
@@ -78,11 +90,15 @@ class DataAdapter(ABC):
 
         # Try to load dataset from local disk
         if os.path.exists(dataset_name_or_path):
-            data_dict = self.load_from_disk(dataset_name_or_path, subset_list, work_dir, **kwargs)
+            logger.info(f'Loading dataset from local disk: {dataset_name_or_path}')
+            data_dict = self.load_from_disk(
+                dataset_name_or_path, subset_list, work_dir, trust_remote_code=False, **kwargs)
         else:
-            data_dict = self.load_from_hub(dataset_name_or_path, subset_list, work_dir, **kwargs)
-        if len(data_dict) == 0 or len(next(iter(data_dict.values()))) == 0:
-            raise ValueError(f'Local dataset is empty: {dataset_name_or_path}')
+            logger.info(f'Loading dataset from hub: {dataset_name_or_path}')
+            data_dict = self.load_from_hub(
+                dataset_name_or_path, subset_list, work_dir, trust_remote_code=True, **kwargs)
+        if len(data_dict) == 0:
+            raise ValueError(f'Dataset is empty: {dataset_name_or_path}')
         return data_dict
 
     def load_from_hub(self, dataset_name_or_path: str, subset_list: list, work_dir: str, **kwargs) -> dict:
@@ -91,8 +107,7 @@ class DataAdapter(ABC):
         datasets_hub: str = kwargs.pop('datasets_hub', HubType.MODELSCOPE)
         split_as_subset: bool = kwargs.pop('split_as_subset', False)
         # Load dataset from remote
-        logger.info(
-            f'Loading dataset from {datasets_hub}: > dataset_name: {dataset_name_or_path} > subsets: {subset_list}')
+        logger.info(f'Loading dataset: dataset_name: {dataset_name_or_path} > subsets: {subset_list}')
 
         data_dict = {}
         split_list = [split for split in [self.train_split, self.eval_split] if split is not None]
@@ -133,21 +148,7 @@ class DataAdapter(ABC):
         If you want to support local dataset, please rewrite this method in xxx_data_adapter.
         Use modelscope.msdatasets.MsDataset.load to load the dataset from local by default.
         """
-        from modelscope.msdatasets import MsDataset
-
-        logger.info(f'Loading dataset from work_dir: {work_dir}: > dataset_name: {dataset_name_or_path} > \
-                subsets: {subset_list}')
-        data_dict = {}
-        subset_list = subset_list or self.subset_list
-        split_list = [split for split in [self.train_split, self.eval_split] if split is not None]
-        for sub_name in subset_list:
-            data_dict[sub_name] = {}
-            # e.g. train: few-shot, test: target dataset to evaluate
-            for split in split_list:
-                dataset = MsDataset.load(
-                    dataset_name=dataset_name_or_path, subset_name=sub_name, split=split, cache_dir=work_dir, **kwargs)
-                data_dict[sub_name].update({split: dataset})
-        return data_dict
+        return self.load_from_hub(dataset_name_or_path, subset_list, work_dir, **kwargs)
 
     def reformat_subset(self, data_dict: dict, subset_key: str, format: str = '{}') -> dict:
         """
@@ -285,6 +286,12 @@ class DataAdapter(ABC):
         kwargs['metric_list'] = self.metric_list
         return ReportGenerator.gen_report(subset_score_map, report_name, **kwargs)
 
+    def gen_prompt_data(self, prompt: str, **kwargs) -> dict:
+        if not isinstance(prompt, list):
+            prompt = [prompt]
+        prompt_data = PromptData(data=prompt, multi_choices=self.choices, system_prompt=self.system_prompt)
+        return prompt_data.to_dict()
+
     def gen_prompt(self, input_d: dict, subset_name: str, few_shot_list: list, **kwargs) -> Any:
         """
         Generate model prompt from raw input, unify the prompt format for different datasets.
@@ -348,3 +355,6 @@ class DataAdapter(ABC):
             The match result. Usually a score (float) for chat/multiple-choice-questions.
         """
         raise NotImplementedError
+
+    def llm_match(self, *args, **kwargs):
+        pass

evalscope/benchmarks/data_collection/data_collection_adapter.py

@@ -15,7 +15,6 @@ logger = get_logger()
 @Benchmark.register(
     name='data_collection',
     dataset_id='',  # dataset_id need to be set
-    model_adapter=ChatGenerationModelAdapter,
     subset_list=['default'],
     metric_list=['AverageAccuracy'],
     few_shot_num=0,

evalscope/benchmarks/general_mcq/general_mcq_adapter.py

@@ -3,9 +3,8 @@ import csv
 import os
 
 from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.constants import EvalType
+from evalscope.constants import EvalType, OutputType
 from evalscope.metrics.metrics import exact_match
-from evalscope.models import MultiChoiceModelAdapter
 from evalscope.utils import ResponseParser
 from evalscope.utils.logger import get_logger
 
@@ -16,8 +15,10 @@ logger = get_logger()
 
 @Benchmark.register(
     name='general_mcq',
+    pretty_name='General MCQ',
     dataset_id='general_mcq',
-    model_adapter=MultiChoiceModelAdapter,
+    model_adapter=OutputType.MULTIPLE_CHOICE,
+    output_types=[OutputType.MULTIPLE_CHOICE, OutputType.GENERATION],
     subset_list=['default'],
     metric_list=['AverageAccuracy'],
     few_shot_num=0,
@@ -27,11 +28,11 @@ logger = get_logger()
     query_template='问题：{question}\n{choices}\n答案: {answer}\n\n')
 class GeneralMCQAdapter(DataAdapter):
 
-    choices = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J']
-
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
 
+        self.choices = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J']
+
     def load_from_disk(self, dataset_name_or_path, subset_list, work_dir, **kwargs) -> dict:
         data_dict = {}
         for subset_name in subset_list:
@@ -85,7 +86,7 @@ class GeneralMCQAdapter(DataAdapter):
 
         full_prompt = self.prompt_template.format(query=context)
 
-        return {'data': [full_prompt], 'multi_choices': self.choices, 'system_prompt': self.system_prompt}
+        return self.gen_prompt_data(full_prompt)
 
     def get_gold_answer(self, input_d: dict) -> str:
         # Get the gold choice
@@ -103,14 +104,10 @@ class GeneralMCQAdapter(DataAdapter):
         Returns:
             The parsed answer. Depending on the dataset. Usually a string for chat.
         """
-        if eval_type == EvalType.CHECKPOINT:
+        if self.model_adapter == OutputType.MULTIPLE_CHOICE:
             return result
-        elif eval_type == EvalType.SERVICE:
-            return ResponseParser.parse_first_option_with_choices(result, self.choices)
-        elif eval_type == EvalType.CUSTOM:
-            return ResponseParser.parse_first_option_with_choices(result, self.choices)
         else:
-            raise ValueError(f'Invalid eval_type: {eval_type}')
+            return ResponseParser.parse_first_option_with_choices(text=result, options=self.choices)
 
     def match(self, gold: str, pred: str) -> float:
         return exact_match(gold=gold, pred=pred)