evalscope 1.0.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

evalscope/benchmarks/math_vision/math_vision_adapter.py

@@ -0,0 +1,116 @@
+# flake8: noqa: E501
+import re
+from typing import Any, Dict, List
+
+from evalscope.api.benchmark import BenchmarkMeta, VisionLanguageAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.evaluator import TaskState
+from evalscope.api.messages import ChatMessageUser, Content, ContentImage, ContentText
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.io_utils import bytes_to_base64
+from evalscope.utils.logger import get_logger
+from evalscope.utils.multi_choices import MultipleChoiceTemplate, parse_answers, prompt
+
+logger = get_logger()
+
+OPEN_PROMPT = '{question}\nPlease reason step by step, and put your final answer within \\boxed{{}} without units.'
+
+MULT_CHOICE_PROMPT = MultipleChoiceTemplate.SINGLE_ANSWER_COT
+
+SUBSET_LIST = ['level 1', 'level 2', 'level 3', 'level 4', 'level 5']
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='math_vision',
+        pretty_name='MathVision',
+        dataset_id='evalscope/MathVision',
+        tags=[Tags.MATH, Tags.REASONING, Tags.MULTIPLE_CHOICE, Tags.MULTI_MODAL],
+        description=
+        'The MATH-Vision (MATH-V) dataset, a meticulously curated collection of 3,040 high-quality mathematical problems with visual contexts sourced from real math competitions.',
+        subset_list=SUBSET_LIST,
+        metric_list=[{
+            'acc': {
+                'numeric': True
+            }
+        }],
+        eval_split='test',
+        prompt_template=OPEN_PROMPT,
+    )
+)
+class MathVisionAdapter(VisionLanguageAdapter):
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.reformat_subset = True
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        if len(record['options']) > 0:
+            question_type = 'multi_choice'
+        else:
+            question_type = 'free_form'
+        content_list, answers_list = MathVisionAdapter.create_content_and_answers_list(record, question_type)
+        metadata = {
+            'id': record['id'],
+            'image': record['image'],
+            'solution': record['solution'],
+            'level': record['level'],
+            'question_type': question_type,
+            'subject': record['subject']
+        }
+        if question_type == 'multi_choice':
+            label_answer = record['answer']
+            return Sample(
+                input=[ChatMessageUser(content=content_list)],
+                choices=answers_list,
+                target=label_answer,
+                subset_key=f'level {record["level"]}',
+                metadata=metadata
+            )
+        elif question_type == 'free_form':
+            return Sample(
+                input=[ChatMessageUser(content=content_list)],
+                target=record['answer'],
+                subset_key=f'level {record["level"]}',
+                metadata=metadata
+            )
+        else:
+            raise ValueError(f'Unexpected question_type: {question_type}')
+
+    @staticmethod
+    def create_content_and_answers_list(record: Dict[str, Any], question_type) -> tuple[List[Content], List[str]]:
+        """
+            Create a list of content elements and a list of answers from a record.
+
+            Args:
+                record (dict): The record containing question, images, and options.
+                question_type (str): The type of this question
+
+
+            Returns:
+                tuple: A tuple containing:
+                    - content_list (list): A list of content elements (text and images).
+                    - answers_list (list): A list of possible answers (for multiple-choice questions).
+        """
+
+        # Replace <image1>, <image2> ... to [image1], [image2], ... from question text
+        question = re.sub(r'<image(\d+)>', r'[image\1]', record['question']).strip()
+
+        if question_type == 'multi_choice':
+            answers_list = record['options']
+            input_text = prompt(question=question, choices=answers_list, template=MULT_CHOICE_PROMPT)
+            content_list: List[Content] = [ContentText(text=input_text)]
+        else:
+            answers_list: List[str] = []
+            content_list: List[Content] = [ContentText(text=OPEN_PROMPT.format(question=question))]
+        image = record['decoded_image']
+        if image:
+            image_base64 = bytes_to_base64(image['bytes'], format='jpg', add_header=True)
+            content_list.append(ContentImage(image=image_base64))
+        return content_list, answers_list
+
+    def extract_answer(self, prediction: str, task_state):
+        from evalscope.metrics.math_parser import extract_answer
+
+        return extract_answer(prediction)

evalscope/benchmarks/math_vista/math_vista_adapter.py

@@ -0,0 +1,114 @@
+# flake8: noqa: E501
+import re
+from typing import Any, Dict
+
+from evalscope.api.benchmark import BenchmarkMeta, VisionLanguageAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.messages import ChatMessageUser, Content, ContentImage, ContentText
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.io_utils import bytes_to_base64
+from evalscope.utils.logger import get_logger
+from evalscope.utils.multi_choices import MultipleChoiceTemplate, parse_answers, prompt
+
+logger = get_logger()
+
+OPEN_PROMPT = '{question}\nPlease reason step by step, and put your final answer within \\boxed{{}} without units.'
+
+MULT_CHOICE_PROMPT = MultipleChoiceTemplate.SINGLE_ANSWER_COT
+
+MULTI_CHOICE_TYPE = 'multi_choice'
+OPEN_TYPE = 'free_form'
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='math_vista',
+        pretty_name='MathVista',
+        dataset_id='evalscope/MathVista',
+        tags=[Tags.MATH, Tags.REASONING, Tags.MULTIPLE_CHOICE, Tags.MULTI_MODAL],
+        description=
+        'MathVista is a consolidated Mathematical reasoning benchmark within Visual contexts. It consists of three newly created datasets, IQTest, FunctionQA, and PaperQA, which address the missing visual domains and are tailored to evaluate logical reasoning on puzzle test figures, algebraic reasoning over functional plots, and scientific reasoning with academic paper figures, respectively. It also incorporates 9 MathQA datasets and 19 VQA datasets from the literature, which significantly enrich the diversity and complexity of visual perception and mathematical reasoning challenges within our benchmark. In total, MathVista includes 6,141 examples collected from 31 different datasets.',
+        metric_list=[{
+            'acc': {
+                'numeric': True
+            }
+        }],
+        eval_split='testmini',
+        prompt_template=OPEN_PROMPT,
+    )
+)
+class MathVistaAdapter(VisionLanguageAdapter):
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        content_list, answers_list = MathVistaAdapter.create_content_and_answers_list(record)
+
+        if record['question_type'] == 'multi_choice':
+            label_answer = self.get_option_label(answers_list, record['answer'])
+            return Sample(
+                input=[ChatMessageUser(content=content_list)],
+                choices=answers_list,
+                target=label_answer,
+                metadata={
+                    'question_type': record['question_type'],
+                    'answer_type': record['answer_type'],
+                    **record['metadata'],
+                }
+            )
+        elif record['question_type'] == 'free_form':
+            return Sample(
+                input=[ChatMessageUser(content=content_list)],
+                target=record['answer'],
+                metadata={
+                    'precision': record['precision'],
+                    'question_type': record['question_type'],
+                    'answer_type': record['answer_type'],
+                    **record['metadata'],
+                }
+            )
+        else:
+            raise ValueError(f"Unexpected question_type: {record['question_type']}")
+
+    def get_option_label(self, options, value):
+        try:
+            index = options.index(value)
+            return chr(ord('A') + index)
+        except ValueError:
+            logger.warning(f"Answer '{value}' not found in options: {options}. This may cause evaluation issues.")
+            return value
+
+    @staticmethod
+    def create_content_and_answers_list(record: dict[str, Any], ) -> tuple[list[Content], list[str]]:
+        """
+            Create a list of content elements and a list of answers from a record.
+
+            Args:
+                record (dict): The record containing question, images, and options.
+
+
+            Returns:
+                tuple: A tuple containing:
+                    - content_list (list): A list of content elements (text and images).
+                    - answers_list (list): A list of possible answers (for multiple-choice questions).
+        """
+        question_type = record['question_type']
+        if question_type == MULTI_CHOICE_TYPE:
+            answers_list = record['choices']
+            input_text = prompt(question=record['question'], choices=answers_list, template=MULT_CHOICE_PROMPT)
+            content_list: list[Content] = [ContentText(text=input_text)]
+        else:
+            answers_list: list[str] = []
+            content_list: list[Content] = [ContentText(text=OPEN_PROMPT.format(question=record['question']))]
+        image = record['decoded_image']
+        if image:
+            image_base64 = bytes_to_base64(image['bytes'], format='jpg', add_header=True)
+            content_list.append(ContentImage(image=image_base64))
+        return content_list, answers_list
+
+    def extract_answer(self, prediction: str, task_state):
+        from evalscope.metrics.math_parser import extract_answer
+
+        return extract_answer(prediction)

evalscope/benchmarks/med_mcqa/med_mcqa_adapter.py

@@ -0,0 +1,32 @@
+from evalscope.api.benchmark import BenchmarkMeta, MultiChoiceAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.multi_choices import MultipleChoiceTemplate
+
+DESCRIPTION = 'MedMCQA is a large-scale MCQA dataset designed to address real-world medical entrance exam questions.'  # noqa: E501
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='med_mcqa',
+        pretty_name='Med-MCQA',
+        tags=[Tags.KNOWLEDGE, Tags.MULTIPLE_CHOICE],
+        description=DESCRIPTION.strip(),
+        dataset_id='extraordinarylab/medmcqa',
+        metric_list=['acc'],
+        few_shot_num=0,
+        train_split='train',
+        eval_split='validation',
+        prompt_template=MultipleChoiceTemplate.SINGLE_ANSWER,
+    )
+)
+class MedMCQAAdapter(MultiChoiceAdapter):
+
+    def record_to_sample(self, record) -> Sample:
+        return Sample(
+            input=record['question'],
+            choices=record['choices'],
+            target=record['answer'],
+            metadata={},
+        )

evalscope/benchmarks/minerva_math/minerva_math_adapter.py

@@ -0,0 +1,53 @@
+from typing import Any, Dict
+
+from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='minerva_math',
+        pretty_name='Minerva-Math',
+        tags=[Tags.MATH, Tags.REASONING],
+        description='Minerva-math is a benchmark designed to evaluate the mathematical and quantitative '
+        'reasoning capabilities of LLMs. It consists of **272 problems** '
+        'sourced primarily from **MIT OpenCourseWare** '
+        'courses, covering advanced STEM subjects such as solid-state chemistry, astronomy, differential '
+        'equations, and special relativity at the **university and graduate level**.',
+        dataset_id='knoveleng/Minerva-Math',
+        subset_list=['default'],
+        metric_list=[{
+            'acc': {
+                'numeric': True
+            }
+        }],
+        eval_split='train',
+        prompt_template='{question}\nPlease reason step by step, and put your final answer within \\boxed{{}}.',
+    )
+)
+class MinervaMathAdapter(DefaultDataAdapter):
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+        self._use_llm_judge = True
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        return Sample(
+            input=record['problem'],
+            target=record['solution'],
+            metadata={
+                'type': record['type'],
+                'idx': record['idx'],
+            },
+        )
+
+    def extract_answer(self, prediction: str, task_state):
+        from evalscope.metrics.math_parser import extract_answer
+
+        return extract_answer(prediction)

evalscope/benchmarks/mm_bench/mm_bench_adapter.py

@@ -0,0 +1,99 @@
+from typing import Any, Dict, List
+
+from evalscope.api.benchmark import BenchmarkMeta, MultiChoiceAdapter, VisionLanguageAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.messages import ChatMessageUser, Content, ContentImage, ContentText
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.io_utils import bytes_to_base64
+from evalscope.utils.logger import get_logger
+from evalscope.utils.multi_choices import MultipleChoiceTemplate, prompt
+
+logger = get_logger()
+
+MULT_CHOICE_PROMPT = MultipleChoiceTemplate.SINGLE_ANSWER_COT
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='cc_bench',
+        pretty_name='CCBench',
+        tags=[Tags.MULTI_MODAL, Tags.KNOWLEDGE, Tags.MULTIPLE_CHOICE],
+        description=
+        'CCBench is an extension of MMBench with newly design questions about Chinese traditional culture, including Calligraphy Painting, Cultural Relic, Food & Clothes, Historical Figures, Scenery & Building, Sketch Reasoning and Traditional Show.',  # noqa: E501
+        dataset_id='lmms-lab/MMBench',
+        subset_list=['cc'],
+        metric_list=['acc'],
+        eval_split='test',
+        prompt_template=MULT_CHOICE_PROMPT,
+    )
+)
+class CCBenchAdapter(VisionLanguageAdapter, MultiChoiceAdapter):
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        answers_list: List[str] = [record.get('A', ''), record.get('B', ''), record.get('C', ''), record.get('D', '')]
+        input_text = prompt(question=record['question'], choices=answers_list, template=self.prompt_template)
+        content_list: List[Content] = [ContentText(text=input_text)]
+        image = record.get('image')
+        if image:
+            image_base64 = bytes_to_base64(image['bytes'], format='jpeg', add_header=True)
+            content_list.append(ContentImage(image=image_base64))
+        label_answer = record.get('answer')
+        return Sample(
+            input=[ChatMessageUser(content=content_list)],
+            choices=answers_list,
+            target=label_answer,
+            metadata={
+                'index': record.get('index'),
+                'category': record.get('category'),
+                'source': record.get('source')
+            }
+        )
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='mm_bench',
+        pretty_name='MMBench',
+        tags=[Tags.MULTI_MODAL, Tags.KNOWLEDGE, Tags.QA],
+        description=
+        'MMBench is a comprehensive evaluation pipeline comprised of meticulously curated multimodal dataset and a novel circulareval strategy using ChatGPT. It is comprised of 20 ability dimensions defined by MMBench. It also contains chinese version with translated question.',  # noqa: E501
+        dataset_id='lmms-lab/MMBench',
+        subset_list=['cn', 'en'],
+        metric_list=['acc'],
+        eval_split='dev',
+        prompt_template=MULT_CHOICE_PROMPT,
+    )
+)
+class MMBenchAdapter(VisionLanguageAdapter, MultiChoiceAdapter):
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        answers_list: List[str] = [record.get('A', ''), record.get('B', ''), record.get('C', ''), record.get('D', '')]
+        answers_list = [ans for ans in answers_list if (ans.strip() and ans != 'nan')]
+        question_hint = record['hint'] + record['question']
+        input_text = prompt(question=question_hint, choices=answers_list, template=self.prompt_template)
+        content_list: List[Content] = [ContentText(text=input_text)]
+        image = record.get('image')
+        if image:
+            image_base64 = bytes_to_base64(image['bytes'], format='jpeg', add_header=True)
+            content_list.append(ContentImage(image=image_base64))
+        label_answer = record.get('answer')
+        return Sample(
+            input=[ChatMessageUser(content=content_list)],
+            choices=answers_list,
+            target=label_answer,
+            metadata={
+                'index': record.get('index'),
+                'category': record.get('category'),
+                'source': record.get('source'),
+                'L2-category': record.get('L2-category'),
+                'comment': record.get('comment'),
+                'split': record.get('split')
+            }
+        )

evalscope/benchmarks/mm_star/mm_star_adapter.py

@@ -0,0 +1,73 @@
+import re
+from typing import Any, Dict, List
+
+from evalscope.api.benchmark import BenchmarkMeta, MultiChoiceAdapter, VisionLanguageAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.evaluator import TaskState
+from evalscope.api.messages import ChatMessageUser, Content, ContentImage, ContentText
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.io_utils import bytes_to_base64
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+MULT_CHOICE_PROMPT = r"""
+Answer the following multiple choice question.
+The last line of your response should be of the following format:
+'ANSWER: $LETTER' (without quotes)
+where LETTER is one of A,B,C,D. Think step by step before answering.
+
+{question}
+""".strip()
+
+SUBSET_LIST = [
+    'coarse perception', 'fine-grained perception', 'instance reasoning', 'logical reasoning', 'math',
+    'science & technology'
+]
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='mm_star',
+        pretty_name='MMStar',
+        tags=[Tags.MULTI_MODAL, Tags.KNOWLEDGE, Tags.MULTIPLE_CHOICE],
+        description=
+        'MMStar: an elite vision-indispensible multi-modal benchmark, aiming to ensure each curated sample exhibits visual dependency, minimal data leakage, and requires advanced multi-modal capabilities.',  # noqa: E501
+        dataset_id='evalscope/MMStar',
+        subset_list=SUBSET_LIST,
+        metric_list=['acc'],
+        default_subset='val',
+        eval_split='val',
+        prompt_template=MULT_CHOICE_PROMPT,
+    )
+)
+class MMStarAdapter(VisionLanguageAdapter, MultiChoiceAdapter):
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+        self.reformat_subset = True
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        input_text = MULT_CHOICE_PROMPT.format(question=record['question'])
+        content_list: List[Content] = [ContentText(text=input_text)]
+        image = record.get('image')
+        if image:
+            image_base64 = bytes_to_base64(image['bytes'], format='jpeg', add_header=True)
+            content_list.append(ContentImage(image=image_base64))
+        label_answer = record.get('answer')
+        return Sample(
+            input=[ChatMessageUser(content=content_list)],
+            choices=['A', 'B', 'C', 'D'],
+            target=label_answer,
+            subset_key=record.get('category'),
+            metadata={
+                'index': record.get('index'),
+                'category': record.get('category'),
+                'l2_category': record.get('l2_category'),
+                'source': record.get('meta_info', {}).get('source'),
+                'split': record.get('meta_info', {}).get('split'),
+                'image_path': record.get('meta_info', {}).get('image_path')
+            }
+        )

evalscope/benchmarks/mmlu_pro/mmlu_pro_adapter.py

@@ -39,7 +39,7 @@ SUBSET_LIST = [
         tags=[Tags.MULTIPLE_CHOICE, Tags.KNOWLEDGE],
         description=
         'MMLU-Pro is a benchmark for evaluating language models on multiple-choice questions across various subjects. It includes questions from different domains, where the model must select the correct answer from given options.',  # noqa: E501
-        dataset_id='modelscope/MMLU-Pro',
+        dataset_id='TIGER-Lab/MMLU-Pro',
         subset_list=SUBSET_LIST,
         metric_list=['acc'],
         few_shot_num=5,

evalscope/benchmarks/mmmu/mmmu_adapter.py

@@ -0,0 +1,159 @@
+import ast
+import re
+from typing import Any, Dict, List
+
+from evalscope.api.benchmark import BenchmarkMeta, VisionLanguageAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.evaluator import TaskState
+from evalscope.api.messages import ChatMessageUser, Content, ContentImage, ContentText
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.io_utils import bytes_to_base64
+from evalscope.utils.logger import get_logger
+from evalscope.utils.multi_choices import MultipleChoiceTemplate, parse_answers, prompt
+
+# flake8: noqa
+
+logger = get_logger()
+
+SUBSET_LIST = [
+    'Accounting',
+    'Agriculture',
+    'Architecture_and_Engineering',
+    'Art',
+    'Art_Theory',
+    'Basic_Medical_Science',
+    'Biology',
+    'Chemistry',
+    'Clinical_Medicine',
+    'Computer_Science',
+    'Design',
+    'Diagnostics_and_Laboratory_Medicine',
+    'Economics',
+    'Electronics',
+    'Energy_and_Power',
+    'Finance',
+    'Geography',
+    'History',
+    'Literature',
+    'Manage',
+    'Marketing',
+    'Materials',
+    'Math',
+    'Mechanical_Engineering',
+    'Music',
+    'Pharmacy',
+    'Physics',
+    'Psychology',
+    'Public_Health',
+    'Sociology',
+]
+
+MULT_CHOICE_PROMPT = MultipleChoiceTemplate.SINGLE_ANSWER_COT
+
+OPEN_PROMPT = """
+Solve the following problem step by step. The last line of your response should be of the form "ANSWER: $ANSWER" (without quotes) where $ANSWER is the answer to the problem.
+
+{question}
+
+Remember to put your answer on its own line at the end in the form "ANSWER: $ANSWER" (without quotes) where $ANSWER is the answer to the problem, and you do not need to use a \\boxed command.
+"""
+
+MULTI_CHOICE_TYPE = 'multiple-choice'
+OPEN_TYPE = 'open'
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='mmmu',
+        pretty_name='MMMU',
+        tags=[Tags.MULTI_MODAL, Tags.KNOWLEDGE, Tags.QA],
+        description=
+        'MMMU (A Massive Multi-discipline Multimodal Understanding and Reasoning Benchmark for Expert AGI) benchmark designed to evaluate multimodal models on massive multi-discipline tasks demanding college-level subject knowledge and deliberate reasoning. MMMU includes 11.5K meticulously collected multimodal questions from college exams, quizzes, and textbooks, covering six core disciplines: Art & Design, Business, Science, Health & Medicine, Humanities & Social Science, and Tech & Engineering. These questions span 30 subjects and 183 subfields, comprising 30 highly heterogeneous image types, such as charts, diagrams, maps, tables, music sheets, and chemical structures.',  # noqa: E501
+        dataset_id='AI-ModelScope/MMMU',
+        subset_list=SUBSET_LIST,
+        metric_list=['acc'],
+        eval_split='validation',
+        prompt_template=OPEN_PROMPT,
+    )
+)
+class MMMUAdapter(VisionLanguageAdapter):
+    MAX_IMAGES: int = 7
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        question_type = record['question_type']
+        content_list, answers_list = MMMUAdapter.create_content_and_answers_list(record)
+
+        metadata = {
+            'id': record['id'],
+            'question_type': record['question_type'],
+            'subfield': record['subfield'],
+            'explanation': record['explanation'],
+            'img_type': record['img_type'],
+            'topic_difficulty': record['topic_difficulty'],
+        }
+
+        if question_type == MULTI_CHOICE_TYPE:
+            return Sample(
+                input=[ChatMessageUser(content=content_list)],
+                choices=answers_list,
+                target=record['answer'],
+                metadata=metadata,
+            )
+        elif question_type == OPEN_TYPE:
+            return Sample(
+                input=[ChatMessageUser(content=content_list)],
+                target=record['answer'],
+                metadata=metadata,
+            )
+        else:
+            raise ValueError(f'Unsupported question type: {question_type}')
+
+    def extract_answer(self, prediction: str, task_state: TaskState) -> str:
+        question_type = task_state.metadata['question_type']
+        if question_type == MULTI_CHOICE_TYPE:
+            answers = parse_answers(task_state)
+            return ''.join(sorted(list(answers)))
+        elif question_type == OPEN_TYPE:
+            pattern = r'ANSWER:\s*(.*)'
+            match = re.search(pattern, prediction)
+            if match:
+                return match.group(1).strip()
+            return prediction.strip()
+        else:
+            raise ValueError(f'Unsupported question type: {question_type}')
+
+    @staticmethod
+    def create_content_and_answers_list(record: Dict[str, Any]) -> tuple[List[Content], List[str]]:
+        """
+        Create a list of content elements and a list of answers from a record.
+
+        Args:
+            record (dict): The record containing question, images, and options.
+
+
+        Returns:
+            tuple: A tuple containing:
+                - content_list (list): A list of content elements (text and images).
+                - answers_list (list): A list of possible answers (for multiple-choice questions).
+        """
+        question_type = record['question_type']
+
+        if question_type == MULTI_CHOICE_TYPE:
+            answers_list: List[str] = ast.literal_eval(record['options'])
+            input_text = prompt(question=record['question'], choices=answers_list, template=MULT_CHOICE_PROMPT)
+            content_list: List[Content] = [ContentText(text=input_text)]
+        else:
+            answers_list: List[str] = []
+            content_list: List[Content] = [ContentText(text=OPEN_PROMPT.format(question=record['question']))]
+
+        for i in range(MMMUAdapter.MAX_IMAGES):
+            image = record[f'image_{i+1}']
+            if image:
+                image_base64 = bytes_to_base64(image['bytes'], format='png', add_header=True)
+                content_list.append(ContentImage(image=image_base64))
+
+        return content_list, answers_list