evalscope 0.17.1__py3-none-any.whl → 1.0.1__py3-none-any.whl

evalscope/benchmarks/mmmu/mmmu_adapter.py

@@ -0,0 +1,159 @@
+import ast
+import re
+from typing import Any, Dict, List
+
+from evalscope.api.benchmark import BenchmarkMeta, VisionLanguageAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.evaluator import TaskState
+from evalscope.api.messages import ChatMessageUser, Content, ContentImage, ContentText
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.io_utils import bytes_to_base64
+from evalscope.utils.logger import get_logger
+from evalscope.utils.multi_choices import MultipleChoiceTemplate, parse_answers, prompt
+
+# flake8: noqa
+
+logger = get_logger()
+
+SUBSET_LIST = [
+    'Accounting',
+    'Agriculture',
+    'Architecture_and_Engineering',
+    'Art',
+    'Art_Theory',
+    'Basic_Medical_Science',
+    'Biology',
+    'Chemistry',
+    'Clinical_Medicine',
+    'Computer_Science',
+    'Design',
+    'Diagnostics_and_Laboratory_Medicine',
+    'Economics',
+    'Electronics',
+    'Energy_and_Power',
+    'Finance',
+    'Geography',
+    'History',
+    'Literature',
+    'Manage',
+    'Marketing',
+    'Materials',
+    'Math',
+    'Mechanical_Engineering',
+    'Music',
+    'Pharmacy',
+    'Physics',
+    'Psychology',
+    'Public_Health',
+    'Sociology',
+]
+
+MULT_CHOICE_PROMPT = MultipleChoiceTemplate.SINGLE_ANSWER_COT
+
+OPEN_PROMPT = """
+Solve the following problem step by step. The last line of your response should be of the form "ANSWER: $ANSWER" (without quotes) where $ANSWER is the answer to the problem.
+
+{question}
+
+Remember to put your answer on its own line at the end in the form "ANSWER: $ANSWER" (without quotes) where $ANSWER is the answer to the problem, and you do not need to use a \\boxed command.
+"""
+
+MULTI_CHOICE_TYPE = 'multiple-choice'
+OPEN_TYPE = 'open'
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='mmmu',
+        pretty_name='MMMU',
+        tags=[Tags.MULTI_MODAL, Tags.KNOWLEDGE, Tags.QA],
+        description=
+        'MMMU (A Massive Multi-discipline Multimodal Understanding and Reasoning Benchmark for Expert AGI) benchmark designed to evaluate multimodal models on massive multi-discipline tasks demanding college-level subject knowledge and deliberate reasoning. MMMU includes 11.5K meticulously collected multimodal questions from college exams, quizzes, and textbooks, covering six core disciplines: Art & Design, Business, Science, Health & Medicine, Humanities & Social Science, and Tech & Engineering. These questions span 30 subjects and 183 subfields, comprising 30 highly heterogeneous image types, such as charts, diagrams, maps, tables, music sheets, and chemical structures.',  # noqa: E501
+        dataset_id='AI-ModelScope/MMMU',
+        subset_list=SUBSET_LIST,
+        metric_list=['acc'],
+        eval_split='validation',
+        prompt_template=OPEN_PROMPT,
+    )
+)
+class MMMUAdapter(VisionLanguageAdapter):
+    MAX_IMAGES: int = 7
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        question_type = record['question_type']
+        content_list, answers_list = MMMUAdapter.create_content_and_answers_list(record)
+
+        metadata = {
+            'id': record['id'],
+            'question_type': record['question_type'],
+            'subfield': record['subfield'],
+            'explanation': record['explanation'],
+            'img_type': record['img_type'],
+            'topic_difficulty': record['topic_difficulty'],
+        }
+
+        if question_type == MULTI_CHOICE_TYPE:
+            return Sample(
+                input=[ChatMessageUser(content=content_list)],
+                choices=answers_list,
+                target=record['answer'],
+                metadata=metadata,
+            )
+        elif question_type == OPEN_TYPE:
+            return Sample(
+                input=[ChatMessageUser(content=content_list)],
+                target=record['answer'],
+                metadata=metadata,
+            )
+        else:
+            raise ValueError(f'Unsupported question type: {question_type}')
+
+    def extract_answer(self, prediction: str, task_state: TaskState) -> str:
+        question_type = task_state.metadata['question_type']
+        if question_type == MULTI_CHOICE_TYPE:
+            answers = parse_answers(task_state)
+            return ''.join(sorted(list(answers)))
+        elif question_type == OPEN_TYPE:
+            pattern = r'ANSWER:\s*(.*)'
+            match = re.search(pattern, prediction)
+            if match:
+                return match.group(1).strip()
+            return ''
+        else:
+            raise ValueError(f'Unsupported question type: {question_type}')
+
+    @staticmethod
+    def create_content_and_answers_list(record: Dict[str, Any]) -> tuple[List[Content], List[str]]:
+        """
+        Create a list of content elements and a list of answers from a record.
+
+        Args:
+            record (dict): The record containing question, images, and options.
+
+
+        Returns:
+            tuple: A tuple containing:
+                - content_list (list): A list of content elements (text and images).
+                - answers_list (list): A list of possible answers (for multiple-choice questions).
+        """
+        question_type = record['question_type']
+
+        if question_type == MULTI_CHOICE_TYPE:
+            answers_list: List[str] = ast.literal_eval(record['options'])
+            input_text = prompt(question=record['question'], choices=answers_list, template=MULT_CHOICE_PROMPT)
+            content_list: List[Content] = [ContentText(text=input_text)]
+        else:
+            answers_list: List[str] = []
+            content_list: List[Content] = [ContentText(text=OPEN_PROMPT.format(question=record['question']))]
+
+        for i in range(MMMUAdapter.MAX_IMAGES):
+            image = record[f'image_{i+1}']
+            if image:
+                image_base64 = bytes_to_base64(image['bytes'], format='png', add_header=True)
+                content_list.append(ContentImage(image=image_base64))
+
+        return content_list, answers_list

evalscope/benchmarks/mmmu_pro/mmmu_pro_adapter.py

@@ -0,0 +1,129 @@
+import ast
+from typing import Any, Dict, List
+
+from evalscope.api.benchmark import BenchmarkMeta, VisionLanguageAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.evaluator import TaskState
+from evalscope.api.messages import ChatMessageUser, Content, ContentImage, ContentText
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.io_utils import bytes_to_base64
+from evalscope.utils.logger import get_logger
+from evalscope.utils.multi_choices import MultipleChoiceTemplate, answer_character, parse_answers, prompt
+
+logger = get_logger()
+
+SUBSET_LIST = [
+    'Accounting',
+    'Agriculture',
+    'Architecture_and_Engineering',
+    'Art',
+    'Art_Theory',
+    'Basic_Medical_Science',
+    'Biology',
+    'Chemistry',
+    'Clinical_Medicine',
+    'Computer_Science',
+    'Design',
+    'Diagnostics_and_Laboratory_Medicine',
+    'Economics',
+    'Electronics',
+    'Energy_and_Power',
+    'Finance',
+    'Geography',
+    'History',
+    'Literature',
+    'Manage',
+    'Marketing',
+    'Materials',
+    'Math',
+    'Mechanical_Engineering',
+    'Music',
+    'Pharmacy',
+    'Physics',
+    'Psychology',
+    'Public_Health',
+    'Sociology',
+]
+
+MULT_CHOICE_PROMPT = MultipleChoiceTemplate.SINGLE_ANSWER_COT
+
+VISION_PROMPT = r"""
+Answer the following multiple choice question in image. The last line of your response should be of the following format: 'ANSWER: $LETTER' (without quotes) where LETTER is one of {letters}. Think step by step before answering.
+
+""".strip()  # noqa: E501
+
+DATASET_FORMATS = ['standard (4 options)', 'standard (10 options)', 'vision']
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='mmmu_pro',
+        pretty_name='MMMU-PRO',
+        tags=[Tags.MULTI_MODAL, Tags.KNOWLEDGE, Tags.QA],
+        description=
+        'MMMU-Pro is an enhanced multimodal benchmark designed to rigorously assess the true understanding capabilities of advanced AI models across multiple modalities. It builds upon the original MMMU benchmark by introducing several key improvements that make it more challenging and realistic, ensuring that models are evaluated on their genuine ability to integrate and comprehend both visual and textual information.',  # noqa: E501
+        dataset_id='AI-ModelScope/MMMU_Pro',
+        subset_list=SUBSET_LIST,
+        metric_list=['acc'],
+        eval_split='test',
+        prompt_template=MULT_CHOICE_PROMPT,
+        extra_params={
+            'dataset_format': f"# choose from {DATASET_FORMATS}, default 'standard (4 options)'",
+        }
+    )
+)
+class MMMUPROAdapter(VisionLanguageAdapter):
+    MAX_IMAGES: int = 7
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        self.reformat_subset = True
+        self.dataset_format = self.extra_params.get('dataset_format', 'standard (4 options)')
+        if self.dataset_format not in DATASET_FORMATS:
+            logger.warning(f"Invalid dataset_format '{self.dataset_format}', fallback to 'standard (4 options)'")
+            self.dataset_format = 'standard (4 options)'
+        self.default_subset = self.dataset_format
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+
+        metadata = {
+            'id': record['id'],
+            'explanation': record.get('explanation'),
+            'img_type': record.get('img_type'),
+            'topic_difficulty': record.get('topic_difficulty'),
+            'subject': record.get('subject')
+        }
+
+        answers_list: List[str] = ast.literal_eval(record['options'])
+
+        if self.dataset_format == 'vision':
+            letters = ','.join(answer_character(i) for i in range(len(answers_list)))
+            input_text = VISION_PROMPT.format(letters=letters)
+            content_list: List[Content] = [ContentText(text=input_text)]
+
+            image = record.get('image')
+            if image:
+                content_list.append(ContentImage(image=bytes_to_base64(image['bytes'], format='png', add_header=True)))
+        else:
+            input_text = prompt(question=record['question'], choices=answers_list, template=MULT_CHOICE_PROMPT)
+            content_list: List[Content] = [ContentText(text=input_text)]
+
+            for i in range(MMMUPROAdapter.MAX_IMAGES):
+                image = record.get(f'image_{i+1}')
+                if image:
+                    image_base64 = bytes_to_base64(image['bytes'], format='png', add_header=True)
+                    content_list.append(ContentImage(image=image_base64))
+
+        return Sample(
+            input=[ChatMessageUser(content=content_list)],
+            choices=answers_list,
+            target=record['answer'],
+            subset_key=record['subject'],
+            metadata=metadata,
+        )
+
+    def extract_answer(self, prediction: str, task_state: TaskState) -> str:
+        answers = parse_answers(task_state)
+        return ''.join(sorted(list(answers)))

evalscope/benchmarks/musr/musr_adapter.py

@@ -1,74 +1,43 @@
 import ast
 from typing import Any
 
-from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.constants import EvalType, OutputType
-from evalscope.metrics import exact_match
-from evalscope.metrics.completion_parsers import ResponseParser
-
-
-@Benchmark.register(
-    name='musr',
-    pretty_name='MuSR',
-    tags=['Reasoning', 'MCQ'],
-    description=
-    'MuSR is a benchmark for evaluating AI models on multiple-choice questions related to murder mysteries, object placements, and team allocation.',  # noqa: E501
-    dataset_id='AI-ModelScope/MuSR',
-    model_adapter=OutputType.GENERATION,
-    output_types=[OutputType.MULTIPLE_CHOICE, OutputType.GENERATION],
-    subset_list=['murder_mysteries', 'object_placements', 'team_allocation'],
-    metric_list=['AverageAccuracy'],
-    few_shot_num=0,
-    train_split=None,
-    eval_split='test',
-    prompt_template=
-    '{narrative}\n\n{question}\n\n{choices}\nThink step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.',  # noqa: E501
+from evalscope.api.benchmark import BenchmarkMeta, MultiChoiceAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.multi_choices import MultipleChoiceTemplate
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='musr',
+        pretty_name='MuSR',
+        tags=[Tags.REASONING, Tags.MULTIPLE_CHOICE],
+        description=
+        'MuSR is a benchmark for evaluating AI models on multiple-choice questions related to murder mysteries, object placements, and team allocation.',  # noqa: E501
+        dataset_id='AI-ModelScope/MuSR',
+        metric_list=['acc'],
+        subset_list=['murder_mysteries', 'object_placements', 'team_allocation'],
+        few_shot_num=0,
+        train_split=None,
+        eval_split='test',
+        prompt_template=MultipleChoiceTemplate.SINGLE_ANSWER_COT,
+    )
 )
-class MuSRAdapter(DataAdapter):
+class MuSRAdapter(MultiChoiceAdapter):
 
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
 
-        self.choices = ['A', 'B', 'C', 'D', 'E', 'F']
-
-    def load(self, **kwargs):
-        # default load all levels
-        kwargs['split_as_subset'] = True
-        data_dict = super().load(**kwargs)
-        return data_dict
-
-    def gen_prompt(self, input_d: dict, subset_name: str, few_shot_list: list, **kwargs) -> Any:
-
-        choices = self.format_choice(ast.literal_eval(input_d['choices']))
-
-        full_prompt = self.prompt_template.format(
-            narrative=input_d['narrative'], question=input_d['question'], choices=choices)
-
-        return self.gen_prompt_data(full_prompt)
-
-    def format_choice(self, options: list):
-        option_str = ''
-        for opt, choice in zip(options, self.choices):
-            option_str += f'({choice}): {opt}\n'
-        return option_str
-
-    def get_gold_answer(self, input_d: dict) -> str:
-        """
-        Parse the raw input labels (gold).
-        """
-        return self.choices[input_d['answer_index']]
+        self.split_as_subset = True
 
-    def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = EvalType.CHECKPOINT) -> str:
-        """
-        Parse the predicted result and extract proper answer.
-        """
-        if self.model_adapter == OutputType.MULTIPLE_CHOICE:
-            return result
-        else:
-            return ResponseParser.parse_first_option(result, options=self.choices)
+    def record_to_sample(self, record) -> Sample:
+        choices = ast.literal_eval(record['choices'])
+        choice_letters = ['A', 'B', 'C', 'D', 'E', 'F']
+        target_letter = choice_letters[record['answer_index']]
 
-    def match(self, gold: str, pred: str) -> float:
-        """
-        Match the gold answer and the predicted answer.
-        """
-        return exact_match(gold=gold, pred=pred)
+        return Sample(
+            input=f"{record['narrative']}\n\n{record['question']}",
+            choices=choices,
+            target=target_letter,
+        )