evalscope 0.17.1__py3-none-any.whl → 1.0.0__py3-none-any.whl

evalscope/benchmarks/ceval/ceval_adapter.py

@@ -1,73 +1,15 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
-import csv
-import os
-from collections import defaultdict
-
-from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.constants import EvalType, OutputType
-from evalscope.metrics import exact_match
-from evalscope.metrics.completion_parsers import ResponseParser
-from evalscope.utils.io_utils import csv_to_list
-from evalscope.utils.logger import get_logger
 
-# flake8: noqa
+from functools import partial
+from typing import Any, Dict
 
-logger = get_logger()
+from evalscope.api.benchmark import BenchmarkMeta, MultiChoiceAdapter
+from evalscope.api.dataset import Dataset, RemoteDataLoader, Sample
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.logger import get_logger
 
-SUBSET_LIST = [
-    'computer_network',
-    'operating_system',
-    'computer_architecture',
-    'college_programming',
-    'college_physics',
-    'college_chemistry',
-    'advanced_mathematics',
-    'probability_and_statistics',
-    'discrete_mathematics',
-    'electrical_engineer',
-    'metrology_engineer',
-    'high_school_mathematics',
-    'high_school_physics',
-    'high_school_chemistry',
-    'high_school_biology',
-    'middle_school_mathematics',
-    'middle_school_biology',
-    'middle_school_physics',
-    'middle_school_chemistry',
-    'veterinary_medicine',
-    'college_economics',
-    'business_administration',
-    'marxism',
-    'mao_zedong_thought',
-    'education_science',
-    'teacher_qualification',
-    'high_school_politics',
-    'high_school_geography',
-    'middle_school_politics',
-    'middle_school_geography',
-    'modern_chinese_history',
-    'ideological_and_moral_cultivation',
-    'logic',
-    'law',
-    'chinese_language_and_literature',
-    'art_studies',
-    'professional_tour_guide',
-    'legal_professional',
-    'high_school_chinese',
-    'high_school_history',
-    'middle_school_history',
-    'civil_servant',
-    'sports_science',
-    'plant_protection',
-    'basic_medicine',
-    'clinical_medicine',
-    'urban_and_rural_planner',
-    'accountant',
-    'fire_engineer',
-    'environmental_impact_assessment_engineer',
-    'tax_accountant',
-    'physician',
-]
+logger = get_logger()
 
 SUBJECT_MAPPING = {
     'computer_network': ['Computer Network', '计算机网络', 'STEM'],
@@ -124,115 +66,105 @@ SUBJECT_MAPPING = {
     'physician': ['Physician', '医师资格', 'Other']
 }
 
-
-@Benchmark.register(
-    name='ceval',
-    pretty_name='C-Eval',
-    tags=['Knowledge', 'MCQ', 'Chinese'],
-    description=
-    'C-Eval is a benchmark designed to evaluate the performance of AI models on Chinese exams across various subjects, including STEM, social sciences, and humanities. It consists of multiple-choice questions that test knowledge and reasoning abilities in these areas.',  # noqa: E501
-    dataset_id='modelscope/ceval-exam',
-    model_adapter=OutputType.GENERATION,
-    output_types=[OutputType.MULTIPLE_CHOICE, OutputType.GENERATION],
-    subset_list=SUBSET_LIST,
-    metric_list=['AverageAccuracy'],
-    few_shot_num=0,
-    train_split='dev',
-    eval_split='val',
-    prompt_template=
-    '以下是中国关于{subset_name}考试的单项选择题，请选出其中的正确答案。你的回答的最后一行应该是这样的格式：“答案是：LETTER”（不带引号），其中 LETTER 是 A、B、C、D 中的一个。\n{query}',
+# Based on the prompt template for Chinese evaluation
+USER_PROMPT_TEMPLATE = """以下是中国关于{subject}的单项选择题，请选出其中的正确答案。你的回答的最后一行应该是这样的格式："答案：LETTER"（不带引号），其中 LETTER 是 A、B、C、D 中的一个。
+
+问题：{question}
+选项：
+{choices}
+""".lstrip()  # noqa: E501
+
+FEWSHOT_TEMPLATE = """以下是一些示例问题：
+
+{fewshot}
+
+""".lstrip()
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='ceval',
+        pretty_name='C-Eval',
+        tags=[Tags.KNOWLEDGE, Tags.MULTIPLE_CHOICE, Tags.CHINESE],
+        description=
+        'C-Eval is a benchmark designed to evaluate the performance of AI models on Chinese exams across various subjects, including STEM, social sciences, and humanities. It consists of multiple-choice questions that test knowledge and reasoning abilities in these areas.',  # noqa: E501
+        dataset_id='evalscope/ceval',
+        subset_list=list(SUBJECT_MAPPING.keys()),
+        metric_list=['acc'],
+        few_shot_num=5,
+        train_split='dev',
+        eval_split='val',
+        prompt_template=USER_PROMPT_TEMPLATE,
+        few_shot_prompt_template=FEWSHOT_TEMPLATE,
+    )
 )
-class CEVALAdapter(DataAdapter):
+class CEVALAdapter(MultiChoiceAdapter):
 
     def __init__(self, **kwargs):
 
-        few_shot_num = kwargs.get('few_shot_num', 0)
-        if few_shot_num > 5:
-            logger.warning(f'few_shot_num <= 5 for C-Eval, but got {few_shot_num}. Use 5-shot by default.')
-            kwargs['few_shot_num'] = 5
         super().__init__(**kwargs)
 
         self.category_map = {k: v[-1] for k, v in SUBJECT_MAPPING.items()}
-        self.choices = ['A', 'B', 'C', 'D']
-
-    def load_from_disk(self, dataset_name_or_path, subset_list, work_dir, **kwargs) -> dict:
-        data_dict = defaultdict(dict)
-        for subset_name in subset_list:
-            for split_name in [self.train_split, self.eval_split]:
-                if os.path.exists(dataset_name_or_path):
-                    file_path = os.path.join(dataset_name_or_path, f'{subset_name}_{split_name}.csv')
-                else:
-                    file_path = os.path.join(work_dir, dataset_name_or_path, f'{subset_name}_{split_name}.csv')
-                if os.path.exists(file_path):
-                    data_dict[subset_name][split_name] = csv_to_list(file_path)
-
-        return data_dict
-
-    def gen_prompt(self, input_d: dict, subset_name: str, few_shot_list: list, **kwargs) -> dict:
-        """
-        Generate model prompt from raw input, unify the prompt format for C-Eval benchmark.
-
-        Args:
-            input_d (dict): The raw input. A single data format of the C-Eval:
-
-            {'id': 0,
-            'question': '下列关于税法基本原则的表述中，不正确的是____。',
-            'A': '税收法定原则包括税收要件法定原则和税务合法性原则',
-            'B': '税收公平原则源于法律上的平等性原则',
-            'C': '税收效率原则包含经济效率和行政效率两个方面',
-            'D': '税务机关按法定程序依法征税，可以自由做出减征、停征或免征税款的决定',
-            'answer': 'D',
-            'explanation': ''}
-
-        Returns:
-            {'data': ['prompt ...']}
-        """
-
-        few_shot_prompts = [self._format_example(input_d=sample, include_answer=True) for sample in few_shot_list]
-
-        if len(few_shot_prompts) > 0:
-            context: str = '\n'.join(few_shot_prompts) + '\n'
-        else:
-            context = ''
-
-        query: str = context.strip() + self._format_example(input_d=input_d, include_answer=False)
-
-        subject_name: str = SUBJECT_MAPPING.get(subset_name)[1] if SUBJECT_MAPPING.get(subset_name) else subset_name
-        full_prompt = self.prompt_template.format(subset_name=subject_name, query=query)
 
-        return self.gen_prompt_data(full_prompt)
-
-    def get_gold_answer(self, input_d: dict) -> str:
-        # Get the gold choice
-        return input_d.get('answer', '')
-
-    def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = EvalType.CHECKPOINT) -> str:
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        # Build choices list from A, B, C, D fields
+        choices = [record['A'], record['B'], record['C'], record['D']]
+        subset = self.current_subset_name
+
+        return Sample(
+            input=record['question'],
+            choices=choices,
+            target=record['answer'],
+            metadata={
+                'id': record.get('id', ''),
+                'explanation': record.get('explanation', ''),
+                'subject': subset
+            },
+        )
+
+    def sample_to_fewshot(self, sample: Sample) -> str:
+        q_str = f"""问题：{sample.input}"""
+        choices = sample.choices if sample.choices is not None else []
+        opt_str_list = []
+        for i, choice in enumerate(choices):
+            opt_str_list.append(f"""{chr(65 + i)}. {choice}""")
+        opt_str = '\n'.join(opt_str_list)
+        opt_str = f"""选项：\n{opt_str}"""
+        exp_str = f"""解析：{sample.metadata.get('explanation', '')}"""
+        ans_str = f"""答案：{sample.target}"""
+        final_str = '\n'.join([q_str, opt_str, exp_str, ans_str])
+
+        return final_str
+
+    def format_fewshot_template(self, fewshot, sample):
+        fewshot_str = FEWSHOT_TEMPLATE.format(fewshot=fewshot)
+        prompt_str = self.format_prompt_template(sample)
+        return fewshot_str + '\n' + prompt_str
+
+    def format_prompt_template(self, sample):
+        subject_name = SUBJECT_MAPPING.get(sample.metadata['subject'])[1]
+        choices = sample.choices if sample.choices is not None else []
+        choices_str = '\n'.join([f'{chr(65 + i)}. {choice}' for i, choice in enumerate(choices)])
+
+        return USER_PROMPT_TEMPLATE.format(subject=subject_name, question=sample.input, choices=choices_str)
+
+    def extract_answer(self, prediction, task_state) -> str:
         """
-        Parse the model output to get the answer. Could be the best choice index.
+        Extract the answer from the prediction based on the task state.
 
         Args:
-            result: Predicted answer from the model. Usually a string for chat.
-            raw_input_d (dict): The raw input. Depending on the dataset.
-            eval_type: `checkpoint` or `service` or `custom`. Default is `checkpoint`.
+            prediction (str): The model's prediction string
+            task_state (dict): The current task state containing metadata
 
         Returns:
-            The parsed answer. Depending on the dataset. Usually a string for chat.
+            str: The extracted answer from the prediction
         """
-        if self.model_adapter == OutputType.MULTIPLE_CHOICE:
-            return result
-        else:
-            return ResponseParser.parse_first_option_with_choices(text=result, options=self.choices)
-
-    def match(self, gold: str, pred: str) -> float:
-        return exact_match(gold=gold, pred=pred)
-
-    def _format_example(self, input_d: dict, include_answer=True):
-        example = '问题：' + input_d['question']
-        for choice in self.choices:
-            example += f'\n{choice}. {input_d[f"{choice}"]}'
+        import re
 
-        if include_answer:
-            example += '\n答案: ' + input_d['answer'] + '\n\n'
+        # Use regex to find the answer in the format "答案：LETTER"
+        match = re.search(r'答案：([A-D])', prediction)
+        if match:
+            return match.group(1)
         else:
-            example += '\n答案: '
-        return example
+            logger.warning(f'No valid answer found in prediction: {prediction}')
+            return ''

evalscope/benchmarks/chinese_simple_qa/csimple_qa_adapter.py

@@ -1,12 +1,14 @@
 import re
-from typing import Any, List
-
-from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.metrics import LLMJudge, Metric, mean, metric_registry
+from typing import Any, Dict
+
+from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.evaluator import TaskState
+from evalscope.api.metric import Score
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
 from evalscope.utils.logger import get_logger
 
-# flake8: noqa
-
 logger = get_logger()
 
 GRADER_TEMPLATE = """
@@ -40,7 +42,7 @@ GRADER_TEMPLATE = """
 模型预测6：你可能是想说贝茜和奥利维亚。不过您应通过最新的参考资料确认详细信息。那是正确的答案吗？
 ```
 这些答复均为【错误】，因为：
-    - 答复中包含与标准答案矛盾的事实陈述。即使在陈述中略带保留（例如：“可能是”，“虽然我不确定，但我认为”），也视为错误。
+    - 答复中包含与标准答案矛盾的事实陈述。即使在陈述中略带保留（例如："可能是"，"虽然我不确定，但我认为"），也视为错误。
 
 以下是【未尝试】的答复示例：
 ```
@@ -56,16 +58,16 @@ GRADER_TEMPLATE = """
     - 回复中没有与标准答案矛盾的陈述。
 
 另外注意以下几点：
-- 对于标准答案为数字的问题，预测答案应和标准答案一致。例如，考虑问题“金山铁路黄浦江特大桥的全长是多少米？”，标准答案为“3518.17”：
-    - 预测答案“3518”、“3518.1”、“3518.17”均为【正确】。
-    - 预测答案“3520”和“3600”均为【错误】。
-    - 预测答案“大约3500米”和“超过3000米”被视为【未尝试】，因为它们既不确认也不与标准答案矛盾。
+- 对于标准答案为数字的问题，预测答案应和标准答案一致。例如，考虑问题"金山铁路黄浦江特大桥的全长是多少米？"，标准答案为"3518.17"：
+    - 预测答案"3518"、"3518.1"、"3518.17"均为【正确】。
+    - 预测答案"3520"和"3600"均为【错误】。
+    - 预测答案"大约3500米"和"超过3000米"被视为【未尝试】，因为它们既不确认也不与标准答案矛盾。
 - 如果标准答案包含比问题更多的信息，预测答案只需包含问题中提到的信息。
-    - 例如，考虑问题“菱镁矿的主要化学成分是什么？”标准答案为“碳酸镁（MgCO3）”。“碳酸镁”或“MgCO3”均视为【正确】答案。
+    - 例如，考虑问题"菱镁矿的主要化学成分是什么？"标准答案为"碳酸镁（MgCO3）"。"碳酸镁"或"MgCO3"均视为【正确】答案。
 - 如果从问题中明显可以推断出预测答案省略的信息，那么算作正确。
-    - 例如，问题“巴鲁米尼的努拉吉遗迹在1997年被联合国教科文组织列为世界文化遗产，那么这遗址在哪个地区？”标准答案为“意大利撒丁岛”，预测答案“撒丁岛”被视为【正确】。
+    - 例如，问题"巴鲁米尼的努拉吉遗迹在1997年被联合国教科文组织列为世界文化遗产，那么这遗址在哪个地区？"标准答案为"意大利撒丁岛"，预测答案"撒丁岛"被视为【正确】。
 - 如果能明显看出名字翻译版本不同但是是同一个人也认为正确。
-    - 例如，如果标准答案是“Robinson”，那么回答鲁滨逊或者鲁滨孙均正确。
+    - 例如，如果标准答案是"Robinson"，那么回答鲁滨逊或者鲁滨孙均正确。
 
 下面是一个新的问题示例。请只回复A、B、C之一，不要道歉或纠正自己的错误，只需要评估该回答。
 ```
@@ -80,88 +82,89 @@ B:【错误】
 C:【未尝试】
 
 只返回字母"A"、"B"或"C"，无须添加其他文本。
-""".strip()  # noqa E501
+""".strip()
 
 SUBSET_LIST = ['中华文化', '人文与社会科学', '工程、技术与应用科学', '生活、艺术与文化', '社会', '自然与自然科学']
 
 
-@Benchmark.register(
-    name='chinese_simpleqa',
-    pretty_name='Chinese-SimpleQA',
-    tags=['Knowledge', 'QA', 'Chinese'],
-    description=
-    "Chinese SimpleQA is a Chinese question-answering dataset designed to evaluate the performance of language models on simple factual questions. It includes a variety of topics and is structured to test the model's ability to understand and generate correct answers in Chinese.",  # noqa: E501
-    subset_list=SUBSET_LIST,
-    dataset_id='AI-ModelScope/Chinese-SimpleQA',
-    metric_list=['is_correct', 'is_incorrect', 'is_not_attempted'],
-    few_shot_num=0,
-    train_split=None,
-    eval_split='train')
-class ChineseSimpleQAAdapter(DataAdapter):
+@register_benchmark(
+    BenchmarkMeta(
+        name='chinese_simpleqa',
+        pretty_name='Chinese-SimpleQA',
+        tags=[Tags.KNOWLEDGE, Tags.QA, Tags.CHINESE],
+        description=
+        "Chinese SimpleQA is a Chinese question-answering dataset designed to evaluate the performance of language models on simple factual questions. It includes a variety of topics and is structured to test the model's ability to understand and generate correct answers in Chinese.",  # noqa: E501
+        subset_list=SUBSET_LIST,
+        dataset_id='AI-ModelScope/Chinese-SimpleQA',
+        metric_list=['is_correct', 'is_incorrect', 'is_not_attempted'],
+        few_shot_num=0,
+        train_split=None,
+        eval_split='train',
+        prompt_template='请回答问题：\n\n{question}'
+    )
+)
+class ChineseSimpleQAAdapter(DefaultDataAdapter):
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
 
-        # register metrics
-        metric_registry.register(Metric(name='is_correct', object=mean))
-        metric_registry.register(Metric(name='is_incorrect', object=mean))
-        metric_registry.register(Metric(name='is_not_attempted', object=mean))
-
-        # whether to use LLM as a judge
-        self.llm_as_a_judge = True
-
-    def load(self, **kwargs):
-        kwargs['subset_list'] = ['default']
-        data_dict = super().load(**kwargs)
-        return self.reformat_subset(data_dict, subset_key='primary_category', format='{}')
-
-    def gen_prompt(self, input_d: dict, subset_name: str, few_shot_list: list, **kwargs) -> dict:
-        question = input_d['question']
-        return self.gen_prompt_data(question)
-
-    def get_gold_answer(self, input_d: dict) -> str:
-        return input_d['answer']
-
-    def parse_pred_result(self, result: str, raw_input_d: dict = None, **kwargs) -> str:
-        return result.strip()
-
-    def match(self, gold: str, pred: str) -> float:
-        # simple match
-        logger.warning(f'Please use LLMJudge to match the result for {self.name}')
-        is_correct = 1 if gold.lower().strip() == pred.lower().strip() else 0
-        is_incorrect = not is_correct
-        is_not_attempted = 0
-        return {
-            'is_correct': is_correct,
-            'is_incorrect': is_incorrect,
-            'is_not_attempted': is_not_attempted,
+        self._use_llm_judge = True  # Use LLM as a judge by default
+        self.reformat_subset = True  # Reformat subset to primary_category
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        """
+        Convert a data record to a Sample object.
+
+        Args:
+            record (Dict[str, Any]): Input data record.
+
+        Returns:
+            Sample: Sample object with input, target, and metadata.
+        """
+        question = record['question']
+        answer = record['answer']
+        subset_key = record.get('primary_category', 'default')  # Use primary_category as subset key
+        metadata = {
+            'id': record.get('id', 'unknown'),
+            'primary_category': subset_key,
+            'secondary_category': record.get('secondary_category', '')
         }
 
-    def llm_match(self, gold: Any, pred: Any, judge: LLMJudge, **kwargs) -> dict:
-        raw_input = kwargs.get('raw_input', None)
-        question = raw_input['question']
-        # get grading response
-        prompt = GRADER_TEMPLATE.format(question=question, target=gold, predicted_answer=pred)
+        return Sample(input=question, target=answer, subset_key=subset_key, metadata=metadata)
+
+    def llm_match_score(
+        self,
+        original_prediction: str,
+        filtered_prediction: str,
+        reference: str,
+        task_state: TaskState,
+    ) -> Score:
+        score = Score(
+            extracted_prediction=filtered_prediction,
+            prediction=original_prediction,
+        )
+
+        question = task_state.input_text
+
+        # Request judge and obtain score
+        prompt = GRADER_TEMPLATE.format(question=question, target=reference, predicted_answer=filtered_prediction)
         system_prompt = '你是一个智能助手，请根据给定问题、标准答案和模型预测的答案来评估模型的回答是否正确。'
-        grading_response = judge(prompt, system_prompt)
+        judge_response = self.llm_judge.judge(prompt, system_prompt=system_prompt)
         # parse grading response
-        match = re.search(r'(A|B|C)', grading_response)
+        match = re.search(r'(A|B|C)', judge_response)
         res = match.group(0) if match else 'C'
-        return {
+
+        # Set score based on the match result
+        score.value = {
             'is_correct': 1 if res == 'A' else 0,
             'is_incorrect': 1 if res == 'B' else 0,
             'is_not_attempted': 1 if res == 'C' else 0,
-            'judge_response': grading_response,
         }
-
-    def compute_metric(self, review_res_list: List[dict], **kwargs) -> List[dict]:
-        """
-        compute weighted mean of the bleu score of all samples
-
-        Args:
-            review_res_list: [{'is_correct': 1, 'is_incorrect': 0, 'is_not_attempted': 0}, ...]
-        """
-        # zip dict answers
-        res_dict = super().compute_dict_metric(review_res_list, **kwargs)
-
-        return super().compute_metric(res_dict, **kwargs)
+        score.explanation = f'LLM judge: {judge_response}'
+        score.metadata = {
+            'source': 'llm_judge',
+            'judge_strategy': self.judge_strategy,
+            'model': self.llm_judge.model_id
+        }
+        score.main_score_name = 'is_correct'
+        return score