evalscope 0.17.1__py3-none-any.whl → 1.0.1__py3-none-any.whl

evalscope/benchmarks/chinese_simple_qa/csimple_qa_adapter.py

@@ -1,12 +1,14 @@
 import re
-from typing import Any, List
-
-from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.metrics import LLMJudge, Metric, mean, metric_registry
+from typing import Any, Dict
+
+from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.evaluator import TaskState
+from evalscope.api.metric import Score
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
 from evalscope.utils.logger import get_logger
 
-# flake8: noqa
-
 logger = get_logger()
 
 GRADER_TEMPLATE = """
@@ -40,7 +42,7 @@ GRADER_TEMPLATE = """
 模型预测6：你可能是想说贝茜和奥利维亚。不过您应通过最新的参考资料确认详细信息。那是正确的答案吗？
 ```
 这些答复均为【错误】，因为：
-    - 答复中包含与标准答案矛盾的事实陈述。即使在陈述中略带保留（例如：“可能是”，“虽然我不确定，但我认为”），也视为错误。
+    - 答复中包含与标准答案矛盾的事实陈述。即使在陈述中略带保留（例如："可能是"，"虽然我不确定，但我认为"），也视为错误。
 
 以下是【未尝试】的答复示例：
 ```
@@ -56,16 +58,16 @@ GRADER_TEMPLATE = """
     - 回复中没有与标准答案矛盾的陈述。
 
 另外注意以下几点：
-- 对于标准答案为数字的问题，预测答案应和标准答案一致。例如，考虑问题“金山铁路黄浦江特大桥的全长是多少米？”，标准答案为“3518.17”：
-    - 预测答案“3518”、“3518.1”、“3518.17”均为【正确】。
-    - 预测答案“3520”和“3600”均为【错误】。
-    - 预测答案“大约3500米”和“超过3000米”被视为【未尝试】，因为它们既不确认也不与标准答案矛盾。
+- 对于标准答案为数字的问题，预测答案应和标准答案一致。例如，考虑问题"金山铁路黄浦江特大桥的全长是多少米？"，标准答案为"3518.17"：
+    - 预测答案"3518"、"3518.1"、"3518.17"均为【正确】。
+    - 预测答案"3520"和"3600"均为【错误】。
+    - 预测答案"大约3500米"和"超过3000米"被视为【未尝试】，因为它们既不确认也不与标准答案矛盾。
 - 如果标准答案包含比问题更多的信息，预测答案只需包含问题中提到的信息。
-    - 例如，考虑问题“菱镁矿的主要化学成分是什么？”标准答案为“碳酸镁（MgCO3）”。“碳酸镁”或“MgCO3”均视为【正确】答案。
+    - 例如，考虑问题"菱镁矿的主要化学成分是什么？"标准答案为"碳酸镁（MgCO3）"。"碳酸镁"或"MgCO3"均视为【正确】答案。
 - 如果从问题中明显可以推断出预测答案省略的信息，那么算作正确。
-    - 例如，问题“巴鲁米尼的努拉吉遗迹在1997年被联合国教科文组织列为世界文化遗产，那么这遗址在哪个地区？”标准答案为“意大利撒丁岛”，预测答案“撒丁岛”被视为【正确】。
+    - 例如，问题"巴鲁米尼的努拉吉遗迹在1997年被联合国教科文组织列为世界文化遗产，那么这遗址在哪个地区？"标准答案为"意大利撒丁岛"，预测答案"撒丁岛"被视为【正确】。
 - 如果能明显看出名字翻译版本不同但是是同一个人也认为正确。
-    - 例如，如果标准答案是“Robinson”，那么回答鲁滨逊或者鲁滨孙均正确。
+    - 例如，如果标准答案是"Robinson"，那么回答鲁滨逊或者鲁滨孙均正确。
 
 下面是一个新的问题示例。请只回复A、B、C之一，不要道歉或纠正自己的错误，只需要评估该回答。
 ```
@@ -80,88 +82,89 @@ B:【错误】
 C:【未尝试】
 
 只返回字母"A"、"B"或"C"，无须添加其他文本。
-""".strip()  # noqa E501
+""".strip()
 
 SUBSET_LIST = ['中华文化', '人文与社会科学', '工程、技术与应用科学', '生活、艺术与文化', '社会', '自然与自然科学']
 
 
-@Benchmark.register(
-    name='chinese_simpleqa',
-    pretty_name='Chinese-SimpleQA',
-    tags=['Knowledge', 'QA', 'Chinese'],
-    description=
-    "Chinese SimpleQA is a Chinese question-answering dataset designed to evaluate the performance of language models on simple factual questions. It includes a variety of topics and is structured to test the model's ability to understand and generate correct answers in Chinese.",  # noqa: E501
-    subset_list=SUBSET_LIST,
-    dataset_id='AI-ModelScope/Chinese-SimpleQA',
-    metric_list=['is_correct', 'is_incorrect', 'is_not_attempted'],
-    few_shot_num=0,
-    train_split=None,
-    eval_split='train')
-class ChineseSimpleQAAdapter(DataAdapter):
+@register_benchmark(
+    BenchmarkMeta(
+        name='chinese_simpleqa',
+        pretty_name='Chinese-SimpleQA',
+        tags=[Tags.KNOWLEDGE, Tags.QA, Tags.CHINESE],
+        description=
+        "Chinese SimpleQA is a Chinese question-answering dataset designed to evaluate the performance of language models on simple factual questions. It includes a variety of topics and is structured to test the model's ability to understand and generate correct answers in Chinese.",  # noqa: E501
+        subset_list=SUBSET_LIST,
+        dataset_id='AI-ModelScope/Chinese-SimpleQA',
+        metric_list=['is_correct', 'is_incorrect', 'is_not_attempted'],
+        few_shot_num=0,
+        train_split=None,
+        eval_split='train',
+        prompt_template='请回答问题：\n\n{question}'
+    )
+)
+class ChineseSimpleQAAdapter(DefaultDataAdapter):
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
 
-        # register metrics
-        metric_registry.register(Metric(name='is_correct', object=mean))
-        metric_registry.register(Metric(name='is_incorrect', object=mean))
-        metric_registry.register(Metric(name='is_not_attempted', object=mean))
-
-        # whether to use LLM as a judge
-        self.llm_as_a_judge = True
-
-    def load(self, **kwargs):
-        kwargs['subset_list'] = ['default']
-        data_dict = super().load(**kwargs)
-        return self.reformat_subset(data_dict, subset_key='primary_category', format='{}')
-
-    def gen_prompt(self, input_d: dict, subset_name: str, few_shot_list: list, **kwargs) -> dict:
-        question = input_d['question']
-        return self.gen_prompt_data(question)
-
-    def get_gold_answer(self, input_d: dict) -> str:
-        return input_d['answer']
-
-    def parse_pred_result(self, result: str, raw_input_d: dict = None, **kwargs) -> str:
-        return result.strip()
-
-    def match(self, gold: str, pred: str) -> float:
-        # simple match
-        logger.warning(f'Please use LLMJudge to match the result for {self.name}')
-        is_correct = 1 if gold.lower().strip() == pred.lower().strip() else 0
-        is_incorrect = not is_correct
-        is_not_attempted = 0
-        return {
-            'is_correct': is_correct,
-            'is_incorrect': is_incorrect,
-            'is_not_attempted': is_not_attempted,
+        self._use_llm_judge = True  # Use LLM as a judge by default
+        self.reformat_subset = True  # Reformat subset to primary_category
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        """
+        Convert a data record to a Sample object.
+
+        Args:
+            record (Dict[str, Any]): Input data record.
+
+        Returns:
+            Sample: Sample object with input, target, and metadata.
+        """
+        question = record['question']
+        answer = record['answer']
+        subset_key = record.get('primary_category', 'default')  # Use primary_category as subset key
+        metadata = {
+            'id': record.get('id', 'unknown'),
+            'primary_category': subset_key,
+            'secondary_category': record.get('secondary_category', '')
         }
 
-    def llm_match(self, gold: Any, pred: Any, judge: LLMJudge, **kwargs) -> dict:
-        raw_input = kwargs.get('raw_input', None)
-        question = raw_input['question']
-        # get grading response
-        prompt = GRADER_TEMPLATE.format(question=question, target=gold, predicted_answer=pred)
+        return Sample(input=question, target=answer, subset_key=subset_key, metadata=metadata)
+
+    def llm_match_score(
+        self,
+        original_prediction: str,
+        filtered_prediction: str,
+        reference: str,
+        task_state: TaskState,
+    ) -> Score:
+        score = Score(
+            extracted_prediction=filtered_prediction,
+            prediction=original_prediction,
+        )
+
+        question = task_state.input_text
+
+        # Request judge and obtain score
+        prompt = GRADER_TEMPLATE.format(question=question, target=reference, predicted_answer=filtered_prediction)
         system_prompt = '你是一个智能助手，请根据给定问题、标准答案和模型预测的答案来评估模型的回答是否正确。'
-        grading_response = judge(prompt, system_prompt)
+        judge_response = self.llm_judge.judge(prompt, system_prompt=system_prompt)
         # parse grading response
-        match = re.search(r'(A|B|C)', grading_response)
+        match = re.search(r'(A|B|C)', judge_response)
         res = match.group(0) if match else 'C'
-        return {
+
+        # Set score based on the match result
+        score.value = {
             'is_correct': 1 if res == 'A' else 0,
             'is_incorrect': 1 if res == 'B' else 0,
             'is_not_attempted': 1 if res == 'C' else 0,
-            'judge_response': grading_response,
         }
-
-    def compute_metric(self, review_res_list: List[dict], **kwargs) -> List[dict]:
-        """
-        compute weighted mean of the bleu score of all samples
-
-        Args:
-            review_res_list: [{'is_correct': 1, 'is_incorrect': 0, 'is_not_attempted': 0}, ...]
-        """
-        # zip dict answers
-        res_dict = super().compute_dict_metric(review_res_list, **kwargs)
-
-        return super().compute_metric(res_dict, **kwargs)
+        score.explanation = f'LLM judge: {judge_response}'
+        score.metadata = {
+            'source': 'llm_judge',
+            'judge_strategy': self.judge_strategy,
+            'model': self.llm_judge.model_id
+        }
+        score.main_score_name = 'is_correct'
+        return score

evalscope/benchmarks/cmmlu/cmmlu_adapter.py

@@ -1,36 +1,16 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 
-import csv
-import os
-from collections import defaultdict
-
-from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.constants import EvalType, OutputType
-from evalscope.metrics import exact_match
-from evalscope.metrics.completion_parsers import ResponseParser
-from evalscope.utils.io_utils import csv_to_list
+from evalscope.api.benchmark import BenchmarkMeta, MultiChoiceAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
 from evalscope.utils.logger import get_logger
+from evalscope.utils.multi_choices import MultipleChoiceTemplate
 
 # flake8: noqa
 
 logger = get_logger()
 
-SUBSET_LIST = [
-    'agronomy', 'anatomy', 'ancient_chinese', 'arts', 'astronomy', 'business_ethics', 'chinese_civil_service_exam',
-    'chinese_driving_rule', 'chinese_food_culture', 'chinese_foreign_policy', 'chinese_history', 'chinese_literature',
-    'chinese_teacher_qualification', 'college_actuarial_science', 'college_education', 'college_engineering_hydrology',
-    'college_law', 'college_mathematics', 'college_medical_statistics', 'clinical_knowledge', 'college_medicine',
-    'computer_science', 'computer_security', 'conceptual_physics', 'construction_project_management', 'economics',
-    'education', 'elementary_chinese', 'elementary_commonsense', 'elementary_information_and_technology',
-    'electrical_engineering', 'elementary_mathematics', 'ethnology', 'food_science', 'genetics', 'global_facts',
-    'high_school_biology', 'high_school_chemistry', 'high_school_geography', 'high_school_mathematics',
-    'high_school_physics', 'high_school_politics', 'human_sexuality', 'international_law', 'journalism',
-    'jurisprudence', 'legal_and_moral_basis', 'logical', 'machine_learning', 'management', 'marketing',
-    'marxist_theory', 'modern_chinese', 'nutrition', 'philosophy', 'professional_accounting', 'professional_law',
-    'professional_medicine', 'professional_psychology', 'public_relations', 'security_study', 'sociology',
-    'sports_science', 'traditional_chinese_medicine', 'virology', 'world_history', 'world_religions'
-]
-
 SUBJECT_MAPPING = {
     'agronomy': ['other', 'Other'],
     'anatomy': ['biology', 'STEM'],
@@ -102,112 +82,41 @@ SUBJECT_MAPPING = {
 }
 
 
-@Benchmark.register(
-    name='cmmlu',
-    pretty_name='C-MMLU',
-    tags=['Knowledge', 'MCQ', 'Chinese'],
-    description=
-    'C-MMLU is a benchmark designed to evaluate the performance of AI models on Chinese language tasks, including reading comprehension, text classification, and more.',
-    dataset_id='modelscope/cmmlu',
-    model_adapter=OutputType.GENERATION,
-    output_types=[OutputType.MULTIPLE_CHOICE, OutputType.GENERATION],
-    subset_list=SUBSET_LIST,
-    metric_list=['AverageAccuracy'],
-    few_shot_num=5,
-    train_split='dev',
-    eval_split='test',
-    prompt_template=
-    '以下是关于{subset_name}的单项选择题，请给出正确答案的选项。你的回答的最后一行应该是这样的格式：“答案：LETTER”（不带引号），其中 LETTER 是 A、B、C、D 中的一个。\n{query}',
+@register_benchmark(
+    BenchmarkMeta(
+        name='cmmlu',
+        pretty_name='C-MMLU',
+        tags=[Tags.KNOWLEDGE, Tags.MULTIPLE_CHOICE, Tags.CHINESE],
+        description=
+        'C-MMLU is a benchmark designed to evaluate the performance of AI models on Chinese language tasks, including reading comprehension, text classification, and more.',
+        dataset_id='evalscope/cmmlu',
+        metric_list=['acc'],
+        subset_list=list(SUBJECT_MAPPING.keys()),
+        few_shot_num=0,
+        train_split=None,
+        eval_split='test',
+        prompt_template=MultipleChoiceTemplate.CHINESE_SINGLE_ANSWER_TEMPLATE_COT,
+    )
 )
-class CMMLUAdapter(DataAdapter):
+class CMMLUAdapter(MultiChoiceAdapter):
 
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
 
+        self.reformat_subset = True
         self.category_map = {k: v[-1] for k, v in SUBJECT_MAPPING.items()}
-        self.choices = ['A', 'B', 'C', 'D']
-
-    def load_from_disk(self, dataset_name_or_path, subset_list, work_dir, **kwargs) -> dict:
-        data_dict = defaultdict(dict)
-        for subset_name in subset_list:
-            for split_name in [self.train_split, self.eval_split]:
-                if os.path.exists(dataset_name_or_path):
-                    file_path = os.path.join(dataset_name_or_path, split_name, f'{subset_name}.csv')
-                else:
-                    file_path = os.path.join(work_dir, dataset_name_or_path, split_name, f'{subset_name}.csv')
-                if os.path.exists(file_path):
-                    data_dict[subset_name][split_name] = csv_to_list(file_path)
-
-        return data_dict
-
-    def gen_prompt(self, input_d: dict, subset_name: str, few_shot_list: list, **kwargs) -> dict:
-        """
-        Generate model prompt from raw input, unify the prompt format for CMMLU benchmark.
-
-        Args:
-            input_d (dict): The raw input. A single data format of the CMMLU:
-
-            {'Question': '下列关于重力的说法正确的是',
-            'A': '在地球周围的物体都要受到重力作用，与其运动状态无关',
-            'B': '对某一物体而言，重力的大小是一个恒量，不随物体的地理位置而改变',
-            'C': '重力就是地球对物体的吸引力，重力的方向总是竖直向下',
-            'D': '在地球表面各处的重力方向都是相同的',
-            'Answer': 'A'}
-
-        Returns:
-            {'data': [(context, continuation), ...]}
-
-        """
-        few_shot_prompts = [self._generate_prompt(input_d=sample, include_answer=True) for sample in few_shot_list]
-        context = '\n'.join(few_shot_prompts) + '\n'
-        context += self._generate_prompt(input_d=input_d, include_answer=False)
-
-        full_prompt = self.prompt_template.format(subset_name=self._format_subject(subset_name), query=context.strip())
-
-        return self.gen_prompt_data(full_prompt)
-
-    def get_gold_answer(self, input_d: dict) -> str:
-        # Get the gold choice
-        return input_d.get('Answer', '')
-
-    def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = EvalType.CHECKPOINT) -> str:
-        """
-        Parse the model output to get the answer. Could be the best choice index.
-
-        Args:
-            result: Predicted answer from the model. Usually a string for chat.
-            raw_input_d: The raw input. Depending on the dataset.
-            eval_type: The evaluation type. 'checkpoint', 'service', 'custom'.
-
-        Returns:
-            The parsed answer. Depending on the dataset. Usually a string for chat.
-        """
-        if self.model_adapter == OutputType.MULTIPLE_CHOICE:
-            return result
-        else:
-            return ResponseParser.parse_first_option_with_choices(text=result, options=self.choices)
-
-    def match(self, gold: str, pred: str) -> float:
-        return exact_match(gold=gold, pred=pred)
-
-    def _generate_prompt(self, input_d: dict, include_answer=True) -> str:
-
-        input_choices: list = [input_d['A'], input_d['B'], input_d['C'], input_d['D']]
-
-        example: str = input_d['Question']
-        for j in range(len(self.choices)):
-            example += '\n{}. {}'.format(self.choices[j], input_choices[j])
 
-        example += '\nAnswer:'
-        if include_answer:
-            example += ' {}\n\n'.format(input_d['Answer'])
+    def record_to_sample(self, record) -> Sample:
 
-        return example
+        # choices: ["(A) 农业生产工具","(B) 土地","(C) 劳动力","(D) 资金"]
+        # remove the leading (A), (B), (C), (D)
+        raw_choices = record['choices']
+        choice_list = [choice[3:].strip() for choice in raw_choices]
 
-    @classmethod
-    def _format_subject(cls, subject):
-        l = subject.split('_')
-        s = ''
-        for entry in l:
-            s += ' ' + entry
-        return s
+        return Sample(
+            input=record['question'],
+            choices=choice_list,
+            target=record['answer'][1],  # answer is like "A"
+            subset_key=record['category'],
+            metadata={'subject': record['category']},
+        )

evalscope/benchmarks/competition_math/competition_math_adapter.py

@@ -1,125 +1,73 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 # Copyright (c) EleutherAI, Inc. and its affiliates.
-import glob
-import json
-import os
-from collections import defaultdict
 
-from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.metrics import extract_answer, math_equal, strip_answer_string
+from typing import Any, Dict
+
+from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.evaluator import TaskState
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
 from evalscope.utils.logger import get_logger
 
 # flake8: noqa
 
 logger = get_logger()
 
-
-@Benchmark.register(
-    name='competition_math',
-    pretty_name='MATH',
-    tags=['Mathematics'],
-    description=
-    'The MATH (Mathematics) benchmark is designed to evaluate the mathematical reasoning abilities of AI models through a variety of problem types, including arithmetic, algebra, geometry, and more.',
-    dataset_id='modelscope/competition_math',
-    subset_list=['Level 1', 'Level 2', 'Level 3', 'Level 4', 'Level 5'],
-    metric_list=['AveragePass@1'],
-    few_shot_num=4,
-    train_split=None,
-    eval_split='test',
-    prompt_template='{query}\nPlease reason step by step, and put your final answer within \\boxed{{}}.',
+PROMPT_TEMPLATE = """
+Problem:
+{question}
+
+Please reason step by step, and put your final answer within \\boxed{{}}.
+""".lstrip()
+
+FEWSHOT_TEMPLATE = """
+Here are some examples of how to solve similar problems:
+
+{fewshot}
+""".lstrip() + PROMPT_TEMPLATE
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='competition_math',
+        pretty_name='MATH',
+        tags=[Tags.MATH, Tags.REASONING],
+        description=
+        'The MATH (Mathematics) benchmark is designed to evaluate the mathematical reasoning abilities of AI models through a variety of problem types, including arithmetic, algebra, geometry, and more.',
+        dataset_id='evalscope/competition_math',
+        subset_list=['Level 1', 'Level 2', 'Level 3', 'Level 4', 'Level 5'],
+        metric_list=[{
+            'acc': {
+                'numeric': True
+            }
+        }],
+        few_shot_num=4,
+        train_split='train',
+        eval_split='test',
+        prompt_template=PROMPT_TEMPLATE,
+        few_shot_prompt_template=FEWSHOT_TEMPLATE,
+    )
 )
-class CompetitionMathAdapter(DataAdapter):
-    """ To be tested for all models. """
+class CompetitionMathAdapter(DefaultDataAdapter):
 
     def __init__(self, **kwargs):
-
-        few_shot_num = kwargs.get('few_shot_num', 4)
-        if few_shot_num != 4 and few_shot_num != 0:
-            logger.error(f'The MATH benchmark ONLY supports 4-shot by system or 0-shot settings, '
-                         f'but got {few_shot_num}. Use 4-shot by default.')
-            kwargs['few_shot_num'] = 4
-
         super().__init__(**kwargs)
 
-    def load(self, **kwargs):
-        # default load all levels
-        kwargs['subset_list'] = ['default']
-        data_dict = super().load(**kwargs)
-        return self.reformat_subset(data_dict, subset_key='level')
-
-    def load_from_disk(self, dataset_name_or_path, subset_list, work_dir, **kwargs) -> dict:
-        data_dict = defaultdict(dict)
-        for subset_name in subset_list:
-            for split_name in [self.train_split, self.eval_split]:
-                if os.path.exists(dataset_name_or_path):
-                    split_dir = os.path.join(dataset_name_or_path, split_name)
-                else:
-                    split_dir = os.path.join(work_dir, dataset_name_or_path, split_name)
-                split_files = glob.glob(os.path.join(split_dir, '**', '*.json'))
-                split_data = []
-                for file_path in split_files:
-                    if os.path.exists(file_path):
-                        with open(file_path, 'r', encoding='utf-8') as f:
-                            split_data.append(json.load(f))
-                data_dict[subset_name][split_name] = split_data
-
-        return data_dict
-
-    def gen_prompt(self, input_d: dict, few_shot_list: list, **kwargs) -> dict:
-        """
-        Generate the prompt for the model input.
-
-        Args:
-            input_d: raw input dict.
-                {"problem": "How many vertical asymptotes does the graph of $y=\\frac{2}{x^2+x-6}$ have?", "level": "Level 3", "type": "Algebra", "solution": "The denominator of the rational function factors into $x^2+x-6=(x-2)(x+3)$. Since the numerator is always nonzero, there is a vertical asymptote whenever the denominator is $0$, which occurs for $x = 2$ and $x = -3$.  Therefore, the graph has $\\boxed{2}$ vertical asymptotes."}
-
-            few_shot_list:  few shot list. Each item is a raw input dict.
-            **kwargs:
-
-        Returns:
-            {'data': [prompt]}
-        """
-        use_fewshot = self.few_shot_num > 0
-        query = self._generate_prompt(input_d, use_fewshot=use_fewshot)
-        full_prompt = self.prompt_template.format(query=query)
-        return self.gen_prompt_data(full_prompt)
-
-    def get_gold_answer(self, input_d: dict) -> str:
-        # Extract the gold answer from the input dict.
-        return strip_answer_string(extract_answer(input_d['solution']))
-
-    def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = 'checkpoint') -> str:
-        """
-        Parse the model output to get the answer. Could be the best choice index.
-
-        Args:
-            result: Predicted answer from the model. Usually a string for chat.
-            raw_input_d (dict): The raw input. Depending on the dataset.
-            eval_type: 'checkpoint' or 'service' or `custom`
-
-        Returns:
-            The parsed answer. Depending on the dataset. Usually a string for chat.
-        """
-        # Note: Use same extraction method for both of checkpoint/service/custom
-        result = strip_answer_string(extract_answer(result))
-        return result
+        self.reformat_subset = True
 
-    def match(self, gold: str, pred: str) -> float:
-        res = math_equal(pred, gold)
-        return 1.0 if res else 0.0
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        from evalscope.metrics.math_parser import extract_answer
 
-    @classmethod
-    def _generate_prompt(cls, input_d: dict, use_fewshot: bool = True) -> str:
-        problem: str = input_d['problem']
+        return Sample(
+            input=record['problem'],
+            target=extract_answer(record['solution']),
+            subset_key=record['level'],
+            metadata={
+                'reasoning': record.get('solution', ''),
+                'type': record.get('type', ''),
+            },
+        )
 
-        if use_fewshot:
-            # Use 4-shot examples by system
-            context = (
-                'Problem:\nFind the domain of the expression $\\frac{{\sqrt{{x-2}}}}{{\sqrt{{5-x}}}}$.}}\nSolution:\nThe expressions inside each square root must be non-negative. Therefore, $x-2 \ge 0$, so $x\ge2$, and $5 - x \ge 0$, so $x \le 5$. Also, the denominator cannot be equal to zero, so $5-x>0$, which gives $x<5$. Therefore, the domain of the expression is $\\boxed{{[2,5)}}$.\nFinal Answer: The final answer is $[2,5)$. I hope it is correct.\n'
-                'Problem:\nIf $\det \mathbf{{A}} = 2$ and $\det \mathbf{{B}} = 12,$ then find $\det (\mathbf{{A}} \mathbf{{B}}).$\nSolution:\nWe have that $\det (\mathbf{{A}} \mathbf{{B}}) = (\det \mathbf{{A}})(\det \mathbf{{B}}) = (2)(12) = \\boxed{{24}}.$\nFinal Answer: The final answer is $24$. I hope it is correct.\n'
-                'Problem:\nTerrell usually lifts two 20-pound weights 12 times. If he uses two 15-pound weights instead, how many times must Terrell lift them in order to lift the same total weight?\nSolution:\nIf Terrell lifts two 20-pound weights 12 times, he lifts a total of $2\cdot 12\cdot20=480$ pounds of weight. If he lifts two 15-pound weights instead for $n$ times, he will lift a total of $2\cdot15\cdot n=30n$ pounds of weight. Equating this to 480 pounds, we can solve for $n$: \\begin{{align*}} 30n&=480\\\\ \Rightarrow\qquad n&=480/30=\\boxed{{16}} \end{{align*}}\nFinal Answer: The final answer is $16$. I hope it is correct.\n'
-                'Problem:\nIf the system of equations: \\begin{{align*}} 6x-4y&=a,\\\\ 6y-9x &=b. \end{{align*}}has a solution $(x, y)$ where $x$ and $y$ are both nonzero, find $\\frac{{a}}{{b}},$ assuming $b$ is nonzero.\nSolution:\nIf we multiply the first equation by $-\\frac{{3}}{{2}}$, we obtain $$6y-9x=-\\frac{{3}}{{2}}a.$$Since we also know that $6y-9x=b$, we have $$-\\frac{{3}}{{2}}a=b\Rightarrow\\frac{{a}}{{b}}=\\boxed{{-\\frac{{2}}{{3}}}}.$$\nFinal Answer: The final answer is $-\\frac{{2}}{{3}}$. I hope it is correct.\n'
-                f'Problem:\n{problem}\nSolution:\n')
-        else:
-            context = 'Problem:\n' + problem + '\nSolution:\n'
-        return context
+    def sample_to_fewshot(self, sample: Sample) -> str:
+        return f'Problem:\n{sample.input}\nSolution:\n{sample.target}'