evalscope 0.10.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

evalscope/benchmarks/mmlu/mmlu_adapter.py

@@ -1,80 +1,14 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
-import csv
-import os
 
-from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.constants import EvalType
-from evalscope.metrics import AverageAccuracy, exact_match
-from evalscope.models import MultiChoiceModelAdapter
-from evalscope.utils import ResponseParser, normalize_score
+from evalscope.api.benchmark import BenchmarkMeta, MultiChoiceAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
 from evalscope.utils.logger import get_logger
-
-# flake8: noqa
+from evalscope.utils.multi_choices import MultipleChoiceTemplate
 
 logger = get_logger()
 
-DATASET_ID = 'modelscope/mmlu'
-
-SUBSET_LIST = [
-    'high_school_european_history',
-    'business_ethics',
-    'clinical_knowledge',
-    'medical_genetics',
-    'high_school_us_history',
-    'high_school_physics',
-    'high_school_world_history',
-    'virology',
-    'high_school_microeconomics',
-    'econometrics',
-    'college_computer_science',
-    'high_school_biology',
-    'abstract_algebra',
-    'professional_accounting',
-    'philosophy',
-    'professional_medicine',
-    'nutrition',
-    'global_facts',
-    'machine_learning',
-    'security_studies',
-    'public_relations',
-    'professional_psychology',
-    'prehistory',
-    'anatomy',
-    'human_sexuality',
-    'college_medicine',
-    'high_school_government_and_politics',
-    'college_chemistry',
-    'logical_fallacies',
-    'high_school_geography',
-    'elementary_mathematics',
-    'human_aging',
-    'college_mathematics',
-    'high_school_psychology',
-    'formal_logic',
-    'high_school_statistics',
-    'international_law',
-    'high_school_mathematics',
-    'high_school_computer_science',
-    'conceptual_physics',
-    'miscellaneous',
-    'high_school_chemistry',
-    'marketing',
-    'professional_law',
-    'management',
-    'college_physics',
-    'jurisprudence',
-    'world_religions',
-    'sociology',
-    'us_foreign_policy',
-    'high_school_macroeconomics',
-    'computer_security',
-    'moral_scenarios',
-    'moral_disputes',
-    'electrical_engineering',
-    'astronomy',
-    'college_biology',
-]
-
 SUBJECT_MAPPING = {
     'abstract_algebra': ['Abstract Algebra', 'math', 'STEM'],
     'anatomy': ['Anatomy', 'health', 'Other'],
@@ -136,150 +70,38 @@ SUBJECT_MAPPING = {
 }
 
 
-@Benchmark.register(
-    name='mmlu',
-    dataset_id='modelscope/mmlu',
-    model_adapter=MultiChoiceModelAdapter,
-    subset_list=SUBSET_LIST,
-    metric_list=[AverageAccuracy],
-    few_shot_num=5,
-    train_split='train',
-    eval_split='test',
-    prompt_template='',
+@register_benchmark(
+    BenchmarkMeta(
+        name='mmlu',
+        pretty_name='MMLU',
+        tags=[Tags.KNOWLEDGE, Tags.MULTIPLE_CHOICE],
+        description=
+        "The MMLU (Massive Multitask Language Understanding) benchmark is a comprehensive evaluation suite designed to assess the performance of language models across a wide range of subjects and tasks. It includes multiple-choice questions from various domains, such as history, science, mathematics, and more, providing a robust measure of a model's understanding and reasoning capabilities.",  # noqa: E501
+        dataset_id='cais/mmlu',
+        metric_list=['acc'],
+        subset_list=list(SUBJECT_MAPPING.keys()),
+        default_subset='all',
+        few_shot_num=5,
+        train_split='dev',
+        eval_split='test',
+        prompt_template=MultipleChoiceTemplate.SINGLE_ANSWER_COT,
+    )
 )
-class MMLUAdapter(DataAdapter):
-
-    choices = ['A', 'B', 'C', 'D']
+class MMLUAdapter(MultiChoiceAdapter):
 
     def __init__(self, **kwargs):
 
-        few_shot_num = kwargs.get('few_shot_num', 5)
-        if few_shot_num > 5:
-            logger.warning(f'few_shot_num <= 5 for MMLU, but got {few_shot_num}. Use 5-shot by default.')
-            kwargs['few_shot_num'] = 5
-
         super().__init__(**kwargs)
 
+        self.reformat_subset = True
         self.category_map = {k: v[-1] for k, v in SUBJECT_MAPPING.items()}
 
-    def load_from_disk(self, dataset_name_or_path, subset_list, work_dir, **kwargs) -> dict:
-        data_dict = {}
-        for subset_name in subset_list:
-            data_dict[subset_name] = {}
-
-            for split_name in [self.train_split, self.eval_split]:
-                if split_name == 'train':
-                    split_name_suffix = 'dev'
-                elif split_name == 'test':
-                    split_name_suffix = 'test'
-                elif split_name == 'validation':
-                    split_name_suffix = 'val'
-                else:
-                    raise ValueError(f'Invalid split name: {split_name}')
-
-                if os.path.exists(dataset_name_or_path):
-                    file_path = os.path.join(dataset_name_or_path, f'{subset_name}_{split_name_suffix}.csv')
-                else:
-                    file_path = os.path.join(work_dir, dataset_name_or_path, f'{subset_name}_{split_name_suffix}.csv')
-
-                if os.path.exists(file_path):
-                    with open(file_path, encoding='utf-8') as f:
-                        rows = []
-                        reader = csv.reader(f)
-                        for row in reader:
-                            if len(row) != 6:
-                                logger.error(f'Mismatch len of row: {row}, len of row should be 6. Skip this row.')
-                                continue
-                            rows.append({
-                                'input': row[0],
-                                'A': row[1],
-                                'B': row[2],
-                                'C': row[3],
-                                'D': row[4],
-                                'target': row[5],
-                            })
-
-                        data_dict[subset_name].update({split_name: rows})
-
-        return data_dict
-
-    def gen_prompt(self, input_d: dict, subset_name: str, few_shot_list: list, **kwargs) -> dict:
-        """
-        Generate model prompt from raw input, unify the prompt format for MMLU benchmark.
-
-        Args:
-            input_d (dict): The raw input. A single data format of the MMLU:
-
-            {'input': '___________ is based on the idea that customer expectations of the service they will receive shape their perception of the actual service encounter.',
-            'A': 'Service quality.',
-            'B': 'Service action.',
-            'C': 'Service recovery.',
-            'D': 'Service satisfaction.',
-            'target': 'A'}
-
-        Returns:
-            {'data': [full_prompt], 'multi_choices': self.choices}
-
-        """
-        prompt = 'The following are multiple choice questions (with answers) about {}.\n\n'.format(
-            self._format_subject(subset_name))
-        few_shot_prompts = [self._generate_prompt(input_d=sample, include_answer=True) for sample in few_shot_list]
-
-        context: str = '\n'.join(few_shot_prompts) + '\n'
-        context += self._generate_prompt(input_d=input_d, include_answer=False)
-        context = prompt + context
-
-        full_prompt: str = context.strip() + self._generate_prompt(input_d=input_d, include_answer=False)
-
-        return {'data': [full_prompt], 'multi_choices': self.choices, 'system_prompt': self.prompt_template}
-
-    def get_gold_answer(self, input_d: dict) -> str:
-        # Get the gold choice
-        return input_d.get('target', '')
-
-    def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = EvalType.CHECKPOINT) -> str:
-        """
-        Parse the model output to get the answer. Could be the best choice index.
-
-        Args:
-            result: Predicted answer from the model. Usually a string for chat.
-            raw_input_d: The raw input. Depending on the dataset.
-            eval_type: 'checkpoint' or 'service' or 'custom'
-
-        Returns:
-            The parsed answer. Depending on the dataset. Usually a string for chat.
-        """
-        if eval_type == EvalType.CHECKPOINT:
-            return result
-        elif eval_type == EvalType.SERVICE:
-            return ResponseParser.parse_first_option_with_choices(result, self.choices)
-        elif eval_type == EvalType.CUSTOM:
-            return ResponseParser.parse_first_option_with_choices(result, self.choices)
-        else:
-            raise ValueError(f'Invalid eval_type: {eval_type}')
-
-    def match(self, gold: str, pred: str) -> float:
-        return exact_match(gold=gold, pred=pred)
-
-    @classmethod
-    def _generate_prompt(cls, input_d: dict, include_answer=True) -> str:
-
-        input_choices: list = [input_d['A'], input_d['B'], input_d['C'], input_d['D']]
-
-        example: str = input_d['input']
-        for j in range(len(cls.choices)):
-            example += '\n{}. {}'.format(cls.choices[j], input_choices[j])
-
-        example += '\nAnswer:'
-        if include_answer:
-            example += ' {}\n\n'.format(input_d['target'])
-
-        return example
-
-    @classmethod
-    def _format_subject(cls, subject):
-        l = subject.split('_')
-        s = ''
-        for entry in l:
-            s += ' ' + entry
-        return s
+    def record_to_sample(self, record) -> Sample:
+        return Sample(
+            input=record['question'],
+            choices=record['choices'],
+            # converts 0 -> A, 1 -> B, etc.
+            target=('ABCD'[record['answer']]),
+            subset_key=record['subject'],
+            metadata={'subject': record['subject']},
+        )

evalscope/benchmarks/mmlu_pro/mmlu_pro_adapter.py

@@ -1,110 +1,94 @@
-from collections import defaultdict
 from typing import Any, Dict
 
-from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.constants import AnswerKeys, EvalType
-from evalscope.metrics import AverageAccuracy, exact_match
-from evalscope.models import ChatGenerationModelAdapter
-from evalscope.utils.utils import ResponseParser
-
-
-@Benchmark.register(
-    name='mmlu_pro',
-    dataset_id='modelscope/mmlu-pro',
-    model_adapter=ChatGenerationModelAdapter,
-    subset_list=['default'],
-    metric_list=[AverageAccuracy],
-    few_shot_num=5,
-    train_split='validation',
-    eval_split='test',
-    prompt_template=
-    'You are an knowledge expert, you are supposed to answer the multi-choice question to derive your final answer as `The answer is ...`.',  # noqa: E501
+from evalscope.api.benchmark import BenchmarkMeta, MultiChoiceAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+# Based on the prompt provided here:
+# https://github.com/EleutherAI/lm-evaluation-harness/tree/main/lm_eval/tasks/mmlu_pro
+SYSTEM_W_EXAMPLES_PROMPT_TEMPLATE = """
+The following are multiple choice questions (with answers) about {subject}. Think step by step and then finish your answer with 'ANSWER: $LETTER' (without quotes) where LETTER is the correct letter choice.
+
+{examples}
+""".lstrip()  # noqa: E501
+
+# Based on MultipleChoiceTemplate.SINGLE_ANSWER provided in the multiple choice solver:
+# https://github.com/UKGovernmentBEIS/inspect_ai/blob/main/src/inspect_ai/solver/_multiple_choice.py
+USER_PROMPT_TEMPLATE = """Answer the following multiple choice question. The last line of your response should be of the following format: 'ANSWER: $LETTER' (without quotes) where LETTER is one of {letters}. Think step by step before answering.
+
+Question:
+{question}
+Options:
+{choices}
+""".lstrip()  # noqa: E501
+
+SUBSET_LIST = [
+    'computer science', 'math', 'chemistry', 'engineering', 'law', 'biology', 'health', 'physics', 'business',
+    'philosophy', 'economics', 'other', 'psychology', 'history'
+]
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='mmlu_pro',
+        pretty_name='MMLU-Pro',
+        tags=[Tags.MULTIPLE_CHOICE, Tags.KNOWLEDGE],
+        description=
+        'MMLU-Pro is a benchmark for evaluating language models on multiple-choice questions across various subjects. It includes questions from different domains, where the model must select the correct answer from given options.',  # noqa: E501
+        dataset_id='TIGER-Lab/MMLU-Pro',
+        subset_list=SUBSET_LIST,
+        metric_list=['acc'],
+        few_shot_num=5,
+        train_split='validation',
+        eval_split='test',
+        prompt_template=USER_PROMPT_TEMPLATE,
+        few_shot_prompt_template=SYSTEM_W_EXAMPLES_PROMPT_TEMPLATE + USER_PROMPT_TEMPLATE,
+    )
 )
-class MMLUProAdapter(DataAdapter):
+class MMLUProAdapter(MultiChoiceAdapter):
 
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
 
-        self.choices = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J']
-        self.categories = [
-            'computer science', 'math', 'chemistry', 'engineering', 'law', 'biology', 'health', 'physics', 'business',
-            'philosophy', 'economics', 'other', 'psychology', 'history'
-        ]
-
-    def gen_prompts(self, data_dict: dict, **kwargs) -> Dict[str, list]:
-        """
-        Generate model prompt from raw input, unify the prompt format for MMLU-Pro benchmark.
-        Return a dict with category as key and list of prompts as value.
-        """
-
-        data_dict = data_dict[self.subset_list[0]]  # Only one subset for MMLU-Pro
-        fewshot_prompts = self.get_fewshot_examples(data_dict)
-
-        #  Use the category as key to group the prompts
-        res_dict = defaultdict(list)
-        # generate prompts for each test sample
-        for entry in data_dict[self.eval_split]:
-            prefix = fewshot_prompts[entry['category']]
-            query = prefix + 'Q: ' + entry['question'] + '\n' + \
-                self.__form_options(entry['options']) + '\n'
-
-            prompt_d = {'data': [query], 'system_prompt': self.prompt_template, AnswerKeys.RAW_INPUT: entry}
-
-            res_dict[entry['category']].append(prompt_d)
-        return res_dict
-
-    def get_fewshot_examples(self, data_dict: dict):
-        # load 5-shot prompts for each category
-        prompts = {c: '' for c in self.categories}
-        for d in data_dict[self.train_split]:
-            prompts[d['category']] += 'Q:' + ' ' + d['question'] + '\n' + \
-                self.__form_options(d['options']) + '\n' + \
-                d['cot_content'] + '\n\n'
-        return prompts
-
-    def __form_options(self, options: list):
-        option_str = 'Options are:\n'
-        for opt, choice in zip(options, self.choices):
-            option_str += f'({choice}): {opt}' + '\n'
-        return option_str
-
-    def get_gold_answer(self, input_d: dict) -> str:
-        """
-        Parse the raw input labels (gold).
-
-        Args:
-            input_d: input raw data. Depending on the dataset.
-
-        Returns:
-            The parsed input. e.g. gold answer ... Depending on the dataset.
-        """
-        return input_d['answer']
-
-    def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = EvalType.CHECKPOINT) -> str:
-        """
-        Parse the predicted result and extract proper answer.
-
-        Args:
-            result: Predicted answer from the model. Usually a string for chat.
-            raw_input_d: The raw input. Depending on the dataset.
-            eval_type: 'checkpoint' or 'service' or `custom`, default: 'checkpoint'
-
-        Returns:
-            The parsed answer. Depending on the dataset. Usually a string for chat.
-        """
-        return ResponseParser.parse_first_option(result)
-
-    def match(self, gold: str, pred: str) -> float:
-        """
-        Match the gold answer and the predicted answer.
-
-        Args:
-            gold (Any): The golden answer. Usually a string for chat/multiple-choice-questions.
-                        e.g. 'A', extracted from get_gold_answer method.
-            pred (Any): The predicted answer. Usually a string for chat/multiple-choice-questions.
-                        e.g. 'B', extracted from parse_pred_result method.
-
-        Returns:
-            The match result. Usually a score (float) for chat/multiple-choice-questions.
-        """
-        return exact_match(gold=gold, pred=pred)
+        self.reformat_subset = True
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        return Sample(
+            input=record['question'],
+            choices=record['options'],
+            target=record['answer'],
+            subset_key=record['category'].lower(),
+            metadata={
+                'cot_content': record['cot_content'],
+                'subject': record['category'].lower(),
+                'question_id': record['question_id'],
+            },
+        )
+
+    def sample_to_fewshot(self, sample: Sample) -> str:
+        q_str = f"""Question:\n{str(sample.input)}"""
+        options = sample.choices if sample.choices is not None else []
+        opt_str_list = []
+        for i, opt in enumerate(options):
+            opt_str_list.append(f"""{chr(65 + i)} {opt}""")
+        opt_str = '\n'.join(opt_str_list)
+        opt_str = f"""Options:\n{opt_str}"""
+        ans_str = sample.metadata['cot_content'] if sample.metadata is not None else ''
+        ans_str = ans_str.replace('The answer is', 'ANSWER:')
+        ans_opt = ans_str.split('ANSWER:')[-1].split('.')[0].strip().strip('(').strip(')')
+        ans_str = ans_str.replace(f'ANSWER: ({ans_opt})', f'ANSWER: {ans_opt}')
+        final_str = '\n'.join([q_str, opt_str, ans_str])
+
+        return final_str
+
+    def format_fewshot_template(self, fewshot, sample):
+        fewshot_str = SYSTEM_W_EXAMPLES_PROMPT_TEMPLATE.format(
+            subject=sample.metadata['subject'],
+            examples=fewshot,
+        )
+        prompt_str = self.format_prompt_template(sample)
+        return fewshot_str + '\n' + prompt_str

evalscope/benchmarks/mmlu_redux/mmlu_redux_adapter.py

@@ -0,0 +1,139 @@
+from typing import Any, Dict
+
+from evalscope.api.benchmark import BenchmarkMeta, MultiChoiceAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.logger import get_logger
+from evalscope.utils.multi_choices import MultipleChoiceTemplate
+
+logger = get_logger()
+
+SUBJECT_MAPPING = {
+    'abstract_algebra': ['Abstract Algebra', 'math', 'STEM'],
+    'anatomy': ['Anatomy', 'health', 'Other'],
+    'astronomy': ['Astronomy', 'physics', 'STEM'],
+    'business_ethics': ['Business Ethics', 'business', 'Other'],
+    'clinical_knowledge': ['Clinical Knowledge', 'health', 'Other'],
+    'college_biology': ['College Biology', 'biology', 'STEM'],
+    'college_chemistry': ['College Chemistry', 'chemistry', 'STEM'],
+    'college_computer_science': ['College Computer Science', 'computer science', 'STEM'],
+    'college_mathematics': ['College Mathematics', 'math', 'STEM'],
+    'college_medicine': ['College Medicine', 'health', 'Other'],
+    'college_physics': ['College Physics', 'physics', 'STEM'],
+    'computer_security': ['Computer Security', 'computer science', 'STEM'],
+    'conceptual_physics': ['Conceptual Physics', 'physics', 'STEM'],
+    'econometrics': ['Econometrics', 'economics', 'Social Science'],
+    'electrical_engineering': ['Electrical Engineering', 'engineering', 'STEM'],
+    'elementary_mathematics': ['Elementary Mathematics', 'math', 'STEM'],
+    'formal_logic': ['Formal Logic', 'philosophy', 'Humanities'],
+    'global_facts': ['Global Facts', 'other', 'Other'],
+    'high_school_biology': ['High School Biology', 'biology', 'STEM'],
+    'high_school_chemistry': ['High School Chemistry', 'chemistry', 'STEM'],
+    'high_school_computer_science': ['High School Computer Science', 'computer science', 'STEM'],
+    'high_school_european_history': ['High School European History', 'history', 'Humanities'],
+    'high_school_geography': ['High School Geography', 'geography', 'Social Science'],
+    'high_school_government_and_politics': ['High School Government And Politics', 'politics', 'Social Science'],
+    'high_school_macroeconomics': ['High School Macroeconomics', 'economics', 'Social Science'],
+    'high_school_mathematics': ['High School Mathematics', 'math', 'STEM'],
+    'high_school_microeconomics': ['High School Microeconomics', 'economics', 'Social Science'],
+    'high_school_physics': ['High School Physics', 'physics', 'STEM'],
+    'high_school_psychology': ['High School Psychology', 'psychology', 'Social Science'],
+    'high_school_statistics': ['High School Statistics', 'math', 'STEM'],
+    'high_school_us_history': ['High School Us History', 'history', 'Humanities'],
+    'high_school_world_history': ['High School World History', 'history', 'Humanities'],
+    'human_aging': ['Human Aging', 'health', 'Other'],
+    'human_sexuality': ['Human Sexuality', 'culture', 'Social Science'],
+    'international_law': ['International Law', 'law', 'Humanities'],
+    'jurisprudence': ['Jurisprudence', 'law', 'Humanities'],
+    'logical_fallacies': ['Logical Fallacies', 'philosophy', 'Humanities'],
+    'machine_learning': ['Machine Learning', 'computer science', 'STEM'],
+    'management': ['Management', 'business', 'Other'],
+    'marketing': ['Marketing', 'business', 'Other'],
+    'medical_genetics': ['Medical Genetics', 'health', 'Other'],
+    'miscellaneous': ['Miscellaneous', 'other', 'Other'],
+    'moral_disputes': ['Moral Disputes', 'philosophy', 'Humanities'],
+    'moral_scenarios': ['Moral Scenarios', 'philosophy', 'Humanities'],
+    'nutrition': ['Nutrition', 'health', 'Other'],
+    'philosophy': ['Philosophy', 'philosophy', 'Humanities'],
+    'prehistory': ['Prehistory', 'history', 'Humanities'],
+    'professional_accounting': ['Professional Accounting', 'other', 'Other'],
+    'professional_law': ['Professional Law', 'law', 'Humanities'],
+    'professional_medicine': ['Professional Medicine', 'health', 'Other'],
+    'professional_psychology': ['Professional Psychology', 'psychology', 'Social Science'],
+    'public_relations': ['Public Relations', 'politics', 'Social Science'],
+    'security_studies': ['Security Studies', 'politics', 'Social Science'],
+    'sociology': ['Sociology', 'culture', 'Social Science'],
+    'us_foreign_policy': ['Us Foreign Policy', 'politics', 'Social Science'],
+    'virology': ['Virology', 'health', 'Other'],
+    'world_religions': ['World Religions', 'philosophy', 'Humanities'],
+}
+
+SUBSET_LIST = list(SUBJECT_MAPPING.keys())
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='mmlu_redux',
+        pretty_name='MMLU-Redux',
+        tags=[Tags.MULTIPLE_CHOICE, Tags.KNOWLEDGE],
+        description=
+        'MMLU-Redux is a benchmark for evaluating language models on multiple-choice questions across various subjects. It includes questions from different domains, where the model must select the correct answer from given options. '  # noqa: E501
+        'The bad answers are corrected.',  # noqa: E501
+        dataset_id='AI-ModelScope/mmlu-redux-2.0',
+        subset_list=SUBSET_LIST,
+        metric_list=[{
+            'acc': {
+                'allow_inclusion': True
+            }
+        }],
+        few_shot_num=0,
+        train_split=None,
+        eval_split='test',
+        prompt_template=MultipleChoiceTemplate.SINGLE_ANSWER_COT,
+    )
+)
+class MMLUReduxAdapter(MultiChoiceAdapter):
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+        if self.few_shot_num > 0:
+            self.few_shot_num = 0
+            logger.warning('Few-shot examples are not supported for MMLU-Redux dataset. Setting few_shot_num to 0.')
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        error_type = record['error_type']
+        choices = record['choices']
+        target_index_list = [int(record['answer'])]
+        correct_answer = record['correct_answer']
+        if error_type == 'no_correct_answer' and correct_answer:
+            choices[target_index_list[0]] = correct_answer
+        elif error_type == 'wrong_groundtruth' and correct_answer:
+            try:
+                target_index_list = [int(correct_answer)]
+            except ValueError:
+                choice_index = ord(correct_answer) - ord('A')
+                target_index_list = [choice_index]
+        elif error_type == 'multiple_correct_answers' and correct_answer:
+            correct_answer = correct_answer.strip('()')
+            try:
+                correct_answer = correct_answer.replace(' and ', ',').replace(' or ', ',')
+                target_index_list = list(map(int, correct_answer.split(',')))
+            except ValueError:
+                try:
+                    target_index_list = [ord(c) - ord('A') for c in correct_answer.split(',')]
+                except TypeError:
+                    # find the index of the correct answer in choices
+                    target_index_list = [choices.index(c) for c in correct_answer.split(',') if c in choices]
+
+        return Sample(
+            input=record['question'],
+            choices=choices,
+            target=['ABCD'[i] for i in target_index_list] if target_index_list else ['A', 'B', 'C', 'D'],
+            metadata={
+                'error_type': error_type,
+                'correct_answer': correct_answer,
+                'potential_reason': record.get('potential_reason', ''),
+            },
+        )