evalscope 0.12.0__py3-none-any.whl → 0.13.0__py3-none-any.whl

evalscope/arguments.py

@@ -1,7 +1,7 @@
 import argparse
 import json
 
-from evalscope.constants import EvalBackend, EvalStage, EvalType
+from evalscope.constants import EvalBackend, EvalStage, EvalType, JudgeStrategy, OutputType
 
 
 class ParseStrArgsAction(argparse.Action):
@@ -73,6 +73,11 @@ def add_argument(parser: argparse.ArgumentParser):
     parser.add_argument('--api-url', type=str, default=None, help='The API url for the remote API model.')
     parser.add_argument('--timeout', type=float, default=None, help='The timeout for the remote API model.')
     parser.add_argument('--stream', action='store_true', default=False, help='Stream mode.')  # noqa: E501
+
+    # LLMJudge arguments
+    parser.add_argument('--judge-strategy', type=str, default=JudgeStrategy.AUTO, help='The judge strategy.')
+    parser.add_argument('--judge-model-args', type=json.loads, default='{}', help='The judge model args, should be a json string.')  # noqa: E501
+    parser.add_argument('--judge-worker-num', type=int, default=8, help='The number of workers for the judge model.')
     # yapf: enable
 
 

evalscope/benchmarks/aime/aime24_adapter.py

@@ -1,6 +1,6 @@
 from evalscope.benchmarks import Benchmark, DataAdapter
+from evalscope.constants import OutputType
 from evalscope.metrics.math_parser import extract_answer, math_equal, strip_answer_string
-from evalscope.models import ChatGenerationModelAdapter
 from evalscope.utils.logger import get_logger
 
 # flake8: noqa
@@ -10,8 +10,8 @@ logger = get_logger()
 
 @Benchmark.register(
     name='aime24',
+    pretty_name='AIME-2024',
     dataset_id='HuggingFaceH4/aime_2024',
-    model_adapter=ChatGenerationModelAdapter,
     subset_list=['default'],
     metric_list=['AveragePass@1'],
     few_shot_num=0,
@@ -31,7 +31,7 @@ class AIME24Adapter(DataAdapter):
         problem = input_d['problem']
         full_prompt = self.prompt_template.format(query=problem)
 
-        return {'data': [full_prompt], 'system_prompt': self.system_prompt}
+        return self.gen_prompt_data(full_prompt)
 
     def get_gold_answer(self, input_d: dict) -> str:
         # Extract the gold answer from the input dict.

evalscope/benchmarks/aime/aime25_adapter.py

@@ -1,6 +1,6 @@
 from evalscope.benchmarks import Benchmark, DataAdapter
+from evalscope.constants import OutputType
 from evalscope.metrics.math_parser import extract_answer, math_equal, strip_answer_string
-from evalscope.models import ChatGenerationModelAdapter
 from evalscope.utils.logger import get_logger
 
 # flake8: noqa
@@ -10,8 +10,8 @@ logger = get_logger()
 
 @Benchmark.register(
     name='aime25',
+    pretty_name='AIME-2025',
     dataset_id='TIGER-Lab/AIME25',
-    model_adapter=ChatGenerationModelAdapter,
     subset_list=['default'],
     metric_list=['AveragePass@1'],
     few_shot_num=0,
@@ -31,7 +31,7 @@ class AIME25Adapter(DataAdapter):
         problem = input_d['question']
         full_prompt = self.prompt_template.format(query=problem)
 
-        return {'data': [full_prompt], 'system_prompt': self.system_prompt}
+        return self.gen_prompt_data(full_prompt)
 
     def get_gold_answer(self, input_d: dict) -> str:
         # Extract the gold answer from the input dict.

evalscope/benchmarks/arc/arc_adapter.py

@@ -4,9 +4,8 @@ import json
 import os
 
 from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.constants import EvalType
+from evalscope.constants import EvalType, OutputType
 from evalscope.metrics import exact_match
-from evalscope.models import MultiChoiceModelAdapter
 from evalscope.utils import ResponseParser
 from evalscope.utils.logger import get_logger
 
@@ -17,19 +16,20 @@ logger = get_logger()
 
 @Benchmark.register(
     name='arc',
+    pretty_name='ARC',
     dataset_id='modelscope/ai2_arc',
-    model_adapter=MultiChoiceModelAdapter,
+    model_adapter=OutputType.MULTIPLE_CHOICE,
+    output_types=[OutputType.MULTIPLE_CHOICE, OutputType.GENERATION],
     subset_list=['ARC-Easy', 'ARC-Challenge'],
     metric_list=['AverageAccuracy'],
     few_shot_num=0,
     train_split='train',
     eval_split='test',
-    prompt_template='',
+    prompt_template=
+    'Given the following question and four candidate answers (A, B, C and D), choose the best answer.\n{query}\nYour response should end with "The best answer is [the_answer_letter]" where the [the_answer_letter] is one of A, B, C or D.',  # noqa
 )
 class ARCAdapter(DataAdapter):
 
-    choices = ['A', 'B', 'C', 'D']
-
     def __init__(self, **kwargs):
         few_shot_num = kwargs.get('few_shot_num', None)
         if few_shot_num is None:
@@ -42,6 +42,8 @@ class ARCAdapter(DataAdapter):
 
         super().__init__(**kwargs)
 
+        self.choices = ['A', 'B', 'C', 'D']
+
     def load_from_disk(self, dataset_name_or_path, subset_list, work_dir, **kwargs) -> dict:
         """
         Load the dataset from local disk.
@@ -60,7 +62,7 @@ class ARCAdapter(DataAdapter):
             for split_name in ['Train', 'Test']:
                 split_path = os.path.join(subset_path, f'{subset_name}-{split_name}.jsonl')
                 if os.path.exists(split_path):
-                    with open(split_path, 'r', errors='ignore') as in_f:
+                    with open(split_path, 'r', errors='ignore', encoding='utf-8') as in_f:
                         rows = []
                         for line in in_f:
                             item = json.loads(line.strip())
@@ -107,12 +109,11 @@ class ARCAdapter(DataAdapter):
             {'data': ['xxx'], 'multi_choices': ['A', 'B', 'C', 'D']}
         """
         few_shot_prompts = [self._generate_prompt(input_d=sample, include_answer=True) for sample in few_shot_list]
-        context: str = '\n'.join(few_shot_prompts)
+        context = '\n'.join(few_shot_prompts) + self._generate_prompt(input_d=input_d, include_answer=False)
 
-        # context = f'The following are multiple choice questions, please output correct answer in the form of A or B or C or D, do not output explanation:\n {context}'
-        full_prompt: str = context + self._generate_prompt(input_d=input_d, include_answer=False)
+        full_prompt = self.prompt_template.format(query=context)
 
-        return {'data': [full_prompt], 'multi_choices': self.choices, 'system_prompt': self.system_prompt}
+        return self.gen_prompt_data(full_prompt)
 
     def get_gold_answer(self, input_d: dict) -> str:
         # Get the gold choice
@@ -130,14 +131,10 @@ class ARCAdapter(DataAdapter):
         Returns:
             The parsed answer. Depending on the dataset. Usually a string for chat.
         """
-        if eval_type == EvalType.CHECKPOINT:
+        if self.model_adapter == OutputType.MULTIPLE_CHOICE:
             return result
-        elif eval_type == EvalType.SERVICE:
-            return ResponseParser.parse_first_option_with_choices(text=result, options=self.choices)
-        elif eval_type == EvalType.CUSTOM:
-            return ResponseParser.parse_first_option_with_choices(text=result, options=self.choices)
         else:
-            raise ValueError(f'Invalid eval_type: {eval_type}')
+            return ResponseParser.parse_first_option(text=result)
 
     def match(self, gold: str, pred: str) -> float:
         return exact_match(gold=gold, pred=pred)
@@ -152,8 +149,8 @@ class ARCAdapter(DataAdapter):
         choices_prompts: str = '\n'.join([label + '. ' + text for text, label in zip(choices_texts, choices_labels)])
         example += '\n' + choices_prompts
 
-        example += '\nAnswer:'
         if include_answer:
+            example += '\nAnswer:'
             example += ' {}\n\n'.format(input_d['answerKey'])
 
         return example

evalscope/benchmarks/bbh/bbh_adapter.py

@@ -8,8 +8,6 @@ import re
 from evalscope.benchmarks import Benchmark, DataAdapter
 from evalscope.constants import AnswerKeys
 from evalscope.metrics import exact_match
-from evalscope.models.chat_adapter import ChatGenerationModelAdapter
-from evalscope.utils import ResponseParser
 from evalscope.utils.logger import get_logger
 
 # flake8: noqa
@@ -60,8 +58,8 @@ SUBSET_LIST = MULTIPLE_CHOICE_LIST + FREE_FORM_LIST
 
 @Benchmark.register(
     name='bbh',
+    pretty_name='BBH',
     dataset_id='modelscope/bbh',
-    model_adapter=ChatGenerationModelAdapter,
     subset_list=SUBSET_LIST,
     metric_list=['AverageAccuracy'],
     few_shot_num=3,
@@ -94,7 +92,7 @@ class BBHAdapter(DataAdapter):
                 else:
                     file_path: str = os.path.join(work_dir, dataset_name_or_path, f'{subset_name}.json')
                 if os.path.exists(file_path):
-                    with open(file_path, 'r') as f:
+                    with open(file_path, 'r', encoding='utf-8') as f:
                         examples = json.load(f)['examples']
                         if subset_name in data_dict:
                             data_dict[subset_name].update({split_name: examples})
@@ -125,7 +123,7 @@ class BBHAdapter(DataAdapter):
             cot_prompts = ''
         full_prompt = cot_prompts + self.prompt_template.format(query=input_d['input'])
 
-        return {'data': [full_prompt], 'system_prompt': self.system_prompt}
+        return self.gen_prompt_data(full_prompt)
 
     def gen_prompts(self, data_dict: dict) -> dict:
         """
@@ -153,7 +151,9 @@ class BBHAdapter(DataAdapter):
         for sub_name, sub_data_dict in data_dict.items():
             few_shot_data = []
             if self.few_shot_num > 0:
-                with open(os.path.join(os.path.dirname(__file__), 'cot_prompts', f'{sub_name}.txt'), 'r') as f:
+                with open(
+                        os.path.join(os.path.dirname(__file__), 'cot_prompts', f'{sub_name}.txt'), 'r',
+                        encoding='utf-8') as f:
                     cot_prompt_str = f.read()
                 few_shot_data = [cot_prompt_str]
 

evalscope/benchmarks/benchmark.py

@@ -1,12 +1,13 @@
 import copy
-from dataclasses import dataclass, field
+from collections import OrderedDict
+from dataclasses import dataclass, field, fields
 from typing import TYPE_CHECKING, Dict, List, Optional
 
+from evalscope.constants import OutputType
+
 if TYPE_CHECKING:
     from evalscope.benchmarks import DataAdapter
 
-from evalscope.models import BaseModelAdapter
-
 BENCHMARK_MAPPINGS = {}
 
 
@@ -15,8 +16,9 @@ class BenchmarkMeta:
     name: str
     dataset_id: str
     data_adapter: 'DataAdapter'
-    model_adapter: BaseModelAdapter
-    subset_list: List[str] = field(default_factory=list)
+    model_adapter: Optional[str] = OutputType.GENERATION
+    output_types: Optional[List[str]] = field(default_factory=lambda: [OutputType.GENERATION])
+    subset_list: List[str] = field(default_factory=lambda: ['default'])
     metric_list: List[str] = field(default_factory=list)
     few_shot_num: int = 0
     few_shot_random: bool = False
@@ -26,6 +28,8 @@ class BenchmarkMeta:
     system_prompt: Optional[str] = None
     query_template: Optional[str] = None
     pretty_name: Optional[str] = None
+    filters: Optional[OrderedDict] = None
+    extra_params: Optional[Dict] = field(default_factory=dict)
 
     def _update(self, args: dict):
         if args.get('local_path'):
@@ -37,12 +41,9 @@ class BenchmarkMeta:
         return self.__dict__
 
     def to_string_dict(self) -> dict:
-        cur_dict = copy.deepcopy(self.__dict__)
+        cur_dict = copy.deepcopy(self.to_dict())
         # cur_dict['data_adapter'] = self.data_adapter.__name__
-        # cur_dict['model_adapter'] = self.model_adapter.__name__
-        # cur_dict['metric_list'] = [metric['name'] for metric in self.metric_list]
         del cur_dict['data_adapter']
-        del cur_dict['model_adapter']
         return cur_dict
 
     def get_data_adapter(self, config: dict = {}) -> 'DataAdapter':
@@ -66,13 +67,13 @@ class Benchmark:
         return benchmark
 
     @classmethod
-    def register(cls, name: str, dataset_id: str, model_adapter: BaseModelAdapter, **kwargs):
+    def register(cls, name: str, dataset_id: str, **kwargs):
 
         def register_wrapper(data_adapter):
             if name in BENCHMARK_MAPPINGS:
                 raise Exception(f'Benchmark {name} already registered')
             BENCHMARK_MAPPINGS[name] = BenchmarkMeta(
-                name=name, data_adapter=data_adapter, model_adapter=model_adapter, dataset_id=dataset_id, **kwargs)
+                name=name, data_adapter=data_adapter, dataset_id=dataset_id, **kwargs)
             return data_adapter
 
         return register_wrapper

evalscope/benchmarks/ceval/ceval_adapter.py

@@ -3,9 +3,8 @@ import csv
 import os
 
 from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.constants import EvalType
+from evalscope.constants import EvalType, OutputType
 from evalscope.metrics.metrics import exact_match
-from evalscope.models import MultiChoiceModelAdapter
 from evalscope.utils import ResponseParser
 from evalscope.utils.logger import get_logger
 
@@ -126,19 +125,20 @@ SUBJECT_MAPPING = {
 
 @Benchmark.register(
     name='ceval',
+    pretty_name='C-Eval',
     dataset_id='modelscope/ceval-exam',
-    model_adapter=MultiChoiceModelAdapter,
+    model_adapter=OutputType.MULTIPLE_CHOICE,
+    output_types=[OutputType.MULTIPLE_CHOICE, OutputType.GENERATION],
     subset_list=SUBSET_LIST,
     metric_list=['AverageAccuracy'],
     few_shot_num=0,
     train_split='dev',
     eval_split='val',
-    prompt_template='以下是中国关于{subset_name}考试的单项选择题，请选出其中的正确答案。\n{query}',
+    prompt_template=
+    '以下是中国关于{subset_name}考试的单项选择题，请选出其中的正确答案。你的回答的最后一行应该是这样的格式：“答案是：LETTER”（不带引号），其中 LETTER 是 A、B、C、D 中的一个。\n{query}',
 )
 class CEVALAdapter(DataAdapter):
 
-    choices = ['A', 'B', 'C', 'D']
-
     def __init__(self, **kwargs):
 
         few_shot_num = kwargs.get('few_shot_num', 0)
@@ -148,6 +148,7 @@ class CEVALAdapter(DataAdapter):
         super().__init__(**kwargs)
 
         self.category_map = {k: v[-1] for k, v in SUBJECT_MAPPING.items()}
+        self.choices = ['A', 'B', 'C', 'D']
 
     def load_from_disk(self, dataset_name_or_path, subset_list, work_dir, **kwargs) -> dict:
         data_dict = {}
@@ -207,7 +208,7 @@ class CEVALAdapter(DataAdapter):
         subject_name: str = SUBJECT_MAPPING.get(subset_name)[1] if SUBJECT_MAPPING.get(subset_name) else subset_name
         full_prompt = self.prompt_template.format(subset_name=subject_name, query=query)
 
-        return {'data': [full_prompt], 'multi_choices': self.choices, 'system_prompt': self.system_prompt}
+        return self.gen_prompt_data(full_prompt)
 
     def get_gold_answer(self, input_d: dict) -> str:
         # Get the gold choice
@@ -225,22 +226,17 @@ class CEVALAdapter(DataAdapter):
         Returns:
             The parsed answer. Depending on the dataset. Usually a string for chat.
         """
-        if eval_type == EvalType.CHECKPOINT:
+        if self.model_adapter == OutputType.MULTIPLE_CHOICE:
             return result
-        elif eval_type == EvalType.SERVICE:
-            return ResponseParser.parse_first_option_with_choices(result, self.choices)
-        elif eval_type == EvalType.CUSTOM:
-            return ResponseParser.parse_first_option_with_choices(result, self.choices)
         else:
-            raise ValueError(f'Invalid eval_type: {eval_type}')
+            return ResponseParser.parse_first_option_with_choices(text=result, options=self.choices)
 
     def match(self, gold: str, pred: str) -> float:
         return exact_match(gold=gold, pred=pred)
 
-    @classmethod
-    def _format_example(cls, input_d: dict, include_answer=True):
+    def _format_example(self, input_d: dict, include_answer=True):
         example = '问题：' + input_d['question']
-        for choice in cls.choices:
+        for choice in self.choices:
             example += f'\n{choice}. {input_d[f"{choice}"]}'
 
         if include_answer:

evalscope/benchmarks/chinese_simple_qa/csimple_qa_adapter.py

@@ -0,0 +1,168 @@
+import re
+from collections import defaultdict
+from typing import Any, List
+
+from evalscope.benchmarks import Benchmark, DataAdapter
+from evalscope.metrics import Metric, mean, metric_registry
+from evalscope.metrics.llm_judge import LLMJudge
+from evalscope.utils.logger import get_logger
+
+# flake8: noqa
+
+logger = get_logger()
+
+GRADER_TEMPLATE = """
+请根据给定问题、标准答案和模型预测的答案来评估模型的回答是否正确。您的任务是将结果评定为：【正确】、【错误】或【未尝试】。
+
+首先，我们将列出每个评定类别的示例，然后请您对新问题的预测答案进行评定。
+以下是【正确】的答复示例：
+```
+问题：贝拉克·奥巴马的孩子叫什么名字？
+标准答案：玛丽亚·奥巴马和萨莎·奥巴马
+模型预测1：Malia Obama and Sasha Obama
+模型预测2：玛丽亚和萨沙
+模型预测3：大多数人会说是玛丽亚和萨莎，但我不确定，需要再确认
+模型预测4：巴拉克·奥巴马有两个女儿，她们分别是玛丽亚·安和娜塔莎·玛丽安，但通常称作玛丽亚·奥巴马和萨莎·奥巴马。玛丽亚出生于1998年7月4日，萨莎出生于2001年6月10日。
+```
+这些答复均为【正确】，因为：
+    - 完整地包含了标准答案中的重要信息。
+    - 不包含任何与标准答案矛盾的信息。
+    - 只关注语义内容，中英文，大小写、标点、语法和顺序不重要。
+    - 答复中出现模糊语句或猜测是可以接受的，前提是包含了标准答案且不含有不正确信息或矛盾。
+
+以下是【错误】的答复示例：
+```
+问题：巴拉克·奥巴马的孩子叫什么名字？
+标准答案：玛丽亚·奥巴马和萨莎·奥巴马
+模型预测1：玛丽亚
+模型预测2：玛丽亚、萨莎和苏珊
+模型预测3：巴拉克·奥巴马没有孩子
+模型预测4：我认为是玛丽亚和萨莎。或者是玛丽亚和杰基。或者是乔伊和玛丽亚。
+模型预测5：虽然我不知道他们的确切名字，但能说出巴拉克·奥巴马有三个孩子。
+模型预测6：你可能是想说贝茜和奥利维亚。不过您应通过最新的参考资料确认详细信息。那是正确的答案吗？
+```
+这些答复均为【错误】，因为：
+    - 答复中包含与标准答案矛盾的事实陈述。即使在陈述中略带保留（例如：“可能是”，“虽然我不确定，但我认为”），也视为错误。
+
+以下是【未尝试】的答复示例：
+```
+问题：巴拉克·奥巴马的孩子叫什么名字？
+标准答案：玛丽亚·奥巴马和萨莎·奥巴马
+模型预测1：我不知道。
+模型预测2：我需要更多关于您所指奥巴马的上下文。
+模型预测3：不查阅网络我无法回答这个问题，不过我知道巴拉克·奥巴马有两个孩子。
+模型预测4：巴拉克·奥巴马有两个孩子。我知道其中一个叫玛丽亚，但我不确定另一个的名字。
+```
+这些答复均为【未尝试】，因为：
+    - 没有包含标准答案中的重要信息。
+    - 回复中没有与标准答案矛盾的陈述。
+
+另外注意以下几点：
+- 对于标准答案为数字的问题，预测答案应和标准答案一致。例如，考虑问题“金山铁路黄浦江特大桥的全长是多少米？”，标准答案为“3518.17”：
+    - 预测答案“3518”、“3518.1”、“3518.17”均为【正确】。
+    - 预测答案“3520”和“3600”均为【错误】。
+    - 预测答案“大约3500米”和“超过3000米”被视为【未尝试】，因为它们既不确认也不与标准答案矛盾。
+- 如果标准答案包含比问题更多的信息，预测答案只需包含问题中提到的信息。
+    - 例如，考虑问题“菱镁矿的主要化学成分是什么？”标准答案为“碳酸镁（MgCO3）”。“碳酸镁”或“MgCO3”均视为【正确】答案。
+- 如果从问题中明显可以推断出预测答案省略的信息，那么算作正确。
+    - 例如，问题“巴鲁米尼的努拉吉遗迹在1997年被联合国教科文组织列为世界文化遗产，那么这遗址在哪个地区？”标准答案为“意大利撒丁岛”，预测答案“撒丁岛”被视为【正确】。
+- 如果能明显看出名字翻译版本不同但是是同一个人也认为正确。
+    - 例如，如果标准答案是“Robinson”，那么回答鲁滨逊或者鲁滨孙均正确。
+
+下面是一个新的问题示例。请只回复A、B、C之一，不要道歉或纠正自己的错误，只需要评估该回答。
+```
+问题: {question}
+正确答案: {target}
+预测答案: {predicted_answer}
+```
+
+将此新问题的预测答案评定为以下之一：
+A:【正确】
+B:【错误】
+C:【未尝试】
+
+只返回字母"A"、"B"或"C"，无须添加其他文本。
+""".strip()  # noqa E501
+
+SUBSET_LIST = ['中华文化', '人文与社会科学', '工程、技术与应用科学', '生活、艺术与文化', '社会', '自然与自然科学']
+
+
+@Benchmark.register(
+    name='chinese_simpleqa',
+    pretty_name='Chinese SimpleQA',
+    subset_list=SUBSET_LIST,
+    dataset_id='AI-ModelScope/Chinese-SimpleQA',
+    metric_list=['is_correct', 'is_incorrect', 'is_not_attempted'],
+    few_shot_num=0,
+    train_split=None,
+    eval_split='train')
+class ChineseSimpleQAAdapter(DataAdapter):
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        # register metrics
+        metric_registry.register(Metric(name='is_correct', object=mean))
+        metric_registry.register(Metric(name='is_incorrect', object=mean))
+        metric_registry.register(Metric(name='is_not_attempted', object=mean))
+
+        # whether to use LLM as a judge
+        self.llm_as_a_judge = True
+
+    def load(self, **kwargs):
+        kwargs['subset_list'] = ['default']
+        data_dict = super().load(**kwargs)
+        return self.reformat_subset(data_dict, subset_key='primary_category', format='{}')
+
+    def gen_prompt(self, input_d: dict, subset_name: str, few_shot_list: list, **kwargs) -> dict:
+        question = input_d['question']
+        return self.gen_prompt_data(question)
+
+    def get_gold_answer(self, input_d: dict) -> str:
+        return input_d['answer']
+
+    def parse_pred_result(self, result: str, raw_input_d: dict = None, **kwargs) -> str:
+        return result.strip()
+
+    def match(self, gold: str, pred: str) -> float:
+        # simple match
+        logger.warning(f'Please use LLMJudge to match the result for ChineseSimpleQA')
+        is_correct = 1 if gold.lower().strip() == pred.lower().strip() else 0
+        is_incorrect = not is_correct
+        is_not_attempted = 0
+        return {
+            'is_correct': is_correct,
+            'is_incorrect': is_incorrect,
+            'is_not_attempted': is_not_attempted,
+        }
+
+    def llm_match(self, gold: Any, pred: Any, judge: LLMJudge, **kwargs) -> dict:
+        raw_input = kwargs.get('raw_input', None)
+        question = raw_input['question']
+        # get grading response
+        prompt = GRADER_TEMPLATE.format(question=question, target=gold, predicted_answer=pred)
+        system_prompt = '你是一个智能助手，请根据给定问题、标准答案和模型预测的答案来评估模型的回答是否正确。'
+        grading_response = judge(prompt, system_prompt)
+        # parse grading response
+        match = re.search(r'(A|B|C)', grading_response)
+        res = match.group(0) if match else 'C'
+        return {
+            'is_correct': 1 if res == 'A' else 0,
+            'is_incorrect': 1 if res == 'B' else 0,
+            'is_not_attempted': 1 if res == 'C' else 0,
+        }
+
+    def compute_metric(self, review_res_list: List[dict], **kwargs) -> List[dict]:
+        """
+        compute weighted mean of the bleu score of all samples
+
+        Args:
+            review_res_list: [{'is_correct': 1, 'is_incorrect': 0, 'is_not_attempted': 0}, ...]
+        """
+        # zip dict answers
+        res_dict = defaultdict(list)
+        for res in review_res_list:
+            for key, value in res.items():
+                res_dict[key].append(value)
+
+        return super().compute_metric(res_dict, **kwargs)

evalscope/benchmarks/cmmlu/cmmlu_adapter.py

@@ -4,9 +4,8 @@ import csv
 import os
 
 from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.constants import EvalType
+from evalscope.constants import EvalType, OutputType
 from evalscope.metrics import exact_match
-from evalscope.models import MultiChoiceModelAdapter
 from evalscope.utils import ResponseParser
 from evalscope.utils.logger import get_logger
 
@@ -103,23 +102,25 @@ SUBJECT_MAPPING = {
 
 @Benchmark.register(
     name='cmmlu',
+    pretty_name='C-MMLU',
     dataset_id='modelscope/cmmlu',
-    model_adapter=MultiChoiceModelAdapter,
+    model_adapter=OutputType.MULTIPLE_CHOICE,
+    output_types=[OutputType.MULTIPLE_CHOICE, OutputType.GENERATION],
     subset_list=SUBSET_LIST,
     metric_list=['AverageAccuracy'],
     few_shot_num=5,
     train_split='dev',
     eval_split='test',
-    prompt_template='以下是关于{subset_name}的单项选择题，请直接给出正确答案的选项。\n{query}',
+    prompt_template=
+    '以下是关于{subset_name}的单项选择题，请给出正确答案的选项。你的回答的最后一行应该是这样的格式：“答案：LETTER”（不带引号），其中 LETTER 是 A、B、C、D 中的一个。\n{query}',
 )
 class CMMLUAdapter(DataAdapter):
 
-    choices = ['A', 'B', 'C', 'D']
-
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
 
         self.category_map = {k: v[-1] for k, v in SUBJECT_MAPPING.items()}
+        self.choices = ['A', 'B', 'C', 'D']
 
     def load_from_disk(self, dataset_name_or_path, subset_list, work_dir, **kwargs) -> dict:
         data_dict = {}
@@ -172,7 +173,7 @@ class CMMLUAdapter(DataAdapter):
 
         full_prompt = self.prompt_template.format(subset_name=self._format_subject(subset_name), query=context.strip())
 
-        return {'data': [full_prompt], 'multi_choices': self.choices, 'system_prompt': self.system_prompt}
+        return self.gen_prompt_data(full_prompt)
 
     def get_gold_answer(self, input_d: dict) -> str:
         # Get the gold choice
@@ -190,26 +191,21 @@ class CMMLUAdapter(DataAdapter):
         Returns:
             The parsed answer. Depending on the dataset. Usually a string for chat.
         """
-        if eval_type == EvalType.CHECKPOINT:
+        if self.model_adapter == OutputType.MULTIPLE_CHOICE:
             return result
-        elif eval_type == EvalType.SERVICE:
-            return ResponseParser.parse_first_option_with_choices(result, self.choices)
-        elif eval_type == EvalType.CUSTOM:
-            return ResponseParser.parse_first_option_with_choices(result, self.choices)
         else:
-            raise ValueError(f'Invalid eval_type: {eval_type}')
+            return ResponseParser.parse_first_option_with_choices(text=result, options=self.choices)
 
     def match(self, gold: str, pred: str) -> float:
         return exact_match(gold=gold, pred=pred)
 
-    @classmethod
-    def _generate_prompt(cls, input_d: dict, include_answer=True) -> str:
+    def _generate_prompt(self, input_d: dict, include_answer=True) -> str:
 
         input_choices: list = [input_d['A'], input_d['B'], input_d['C'], input_d['D']]
 
         example: str = input_d['Question']
-        for j in range(len(cls.choices)):
-            example += '\n{}. {}'.format(cls.choices[j], input_choices[j])
+        for j in range(len(self.choices)):
+            example += '\n{}. {}'.format(self.choices[j], input_choices[j])
 
         example += '\nAnswer:'
         if include_answer:

evalscope/benchmarks/competition_math/competition_math_adapter.py

@@ -18,8 +18,8 @@ logger = get_logger()
 
 @Benchmark.register(
     name='competition_math',
+    pretty_name='MATH',
     dataset_id='modelscope/competition_math',
-    model_adapter=ChatGenerationModelAdapter,
     subset_list=['Level 1', 'Level 2', 'Level 3', 'Level 4', 'Level 5'],
     metric_list=['AveragePass@1'],
     few_shot_num=4,
@@ -58,7 +58,7 @@ class CompetitionMathAdapter(DataAdapter):
                 split_data = []
                 for file_path in split_files:
                     if os.path.exists(file_path):
-                        with open(file_path, 'r') as f:
+                        with open(file_path, 'r', encoding='utf-8') as f:
                             split_data.append(json.load(f))
                 data_dict[subset_name][split_name] = split_data
 
@@ -81,7 +81,7 @@ class CompetitionMathAdapter(DataAdapter):
         use_fewshot = self.few_shot_num > 0
         query = self._generate_prompt(input_d, use_fewshot=use_fewshot)
         full_prompt = self.prompt_template.format(query=query)
-        return {'data': [full_prompt], 'system_prompt': self.system_prompt}
+        return self.gen_prompt_data(full_prompt)
 
     def get_gold_answer(self, input_d: dict) -> str:
         # Extract the gold answer from the input dict.