evalscope 0.10.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

evalscope/benchmarks/pumed_qa/pubmed_qa_adapter.py

@@ -0,0 +1,175 @@
+import re
+from typing import Any, Dict, List
+
+from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.messages import ChatMessageUser, Content, ContentText
+from evalscope.api.metric.scorer import AggScore, SampleScore, Score
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+DESCRIPTION = ('PubMedQA reasons over biomedical research texts to answer the multiple-choice questions.')
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='pubmedqa',
+        pretty_name='PubMedQA',
+        tags=[Tags.KNOWLEDGE, Tags.YES_NO],
+        description=DESCRIPTION.strip(),
+        dataset_id='extraordinarylab/pubmed-qa',
+        metric_list=['accuracy', 'precision', 'recall', 'f1_score', 'yes_ratio', 'maybe_ratio'],
+        aggregation='f1',
+        few_shot_num=0,
+        eval_split='test',
+        prompt_template='{question}\nPlease answer YES or NO or MAYBE without an explanation.',
+    )
+)
+class PubMedQAAdapter(DefaultDataAdapter):
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.add_overall_metric = False
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        abstract = record['context']
+        question = record['question']
+        question = f'Abstract: {abstract}\n\nQuestion: {question}'
+        input_text = self.prompt_template.format(question=question)
+        content_list: List[Content] = [ContentText(text=input_text)]
+        answer = str(record['answer']).upper()  # 'YES' or 'NO' or 'MAYBE'
+        return Sample(
+            input=[ChatMessageUser(content=content_list)],
+            target=answer,
+            metadata={
+                'answer': record['answer'],
+                'reasoning': record['reasoning'],
+            }
+        )
+
+    def match_score(self, original_prediction, filtered_prediction, reference, task_state) -> Score:
+        score = Score(
+            extracted_prediction=filtered_prediction,
+            prediction=original_prediction,
+        )
+        # Check if the reference answer is in the filtered prediction
+        result = 1 if re.search(r'\b' + re.escape(reference) + r'\b', filtered_prediction.strip().upper()) else 0
+        score.value = {'acc': result}
+        return score
+
+    def aggregate_scores(self, sample_scores: List[SampleScore]) -> List[AggScore]:
+        """
+        Custom aggregation to compute accuracy, precision, recall, f1_score, yes_ratio and maybe_ratio.
+        Handles multi-class classification with YES, NO, and MAYBE answers.
+        """
+
+        def compute_metrics(scores: List[SampleScore]):
+            # Initialize confusion matrix for multi-class classification
+            confusion_matrix = {
+                'YES': {
+                    'YES': 0,
+                    'NO': 0,
+                    'MAYBE': 0
+                },
+                'NO': {
+                    'YES': 0,
+                    'NO': 0,
+                    'MAYBE': 0
+                },
+                'MAYBE': {
+                    'YES': 0,
+                    'NO': 0,
+                    'MAYBE': 0
+                }
+            }
+
+            yes_count = 0
+            maybe_count = 0
+            total_count = len(scores)
+            correct_count = 0
+
+            for ss in scores:
+                gt = ss.sample_metadata['answer'].strip().upper()
+
+                if ss.score.main_value == 1:
+                    correct_count += 1
+                    pred = gt
+                else:
+                    pred_text = ss.score.extracted_prediction.strip().upper()
+                    # Heuristic to determine the predicted class from text
+                    if 'YES' in pred_text:
+                        pred = 'YES'
+                    elif 'NO' in pred_text:
+                        pred = 'NO'
+                    elif 'MAYBE' in pred_text:
+                        pred = 'MAYBE'
+                    else:
+                        pred = None
+
+                if pred:
+                    if pred == 'YES':
+                        yes_count += 1
+                    elif pred == 'MAYBE':
+                        maybe_count += 1
+
+                    if gt in confusion_matrix and pred in confusion_matrix[gt]:
+                        confusion_matrix[gt][pred] += 1
+
+            # Calculate accuracy
+            accuracy = correct_count / total_count if total_count > 0 else 0.0
+
+            # Calculate per-class precision, recall, and F1
+            classes = ['YES', 'NO', 'MAYBE']
+            precision_values = []
+            recall_values = []
+            f1_values = []
+
+            for cls in classes:
+                # True positives for this class
+                tp = confusion_matrix[cls][cls]
+
+                # Calculate predicted positives (column sum)
+                pred_pos = sum(confusion_matrix[true_cls][cls] for true_cls in classes)
+
+                # Calculate actual positives (row sum)
+                act_pos = sum(confusion_matrix[cls][pred_cls] for pred_cls in classes)
+
+                # Calculate precision and recall for this class
+                cls_precision = tp / pred_pos if pred_pos > 0 else 0.0
+                cls_recall = tp / act_pos if act_pos > 0 else 0.0
+
+                # Calculate F1 for this class
+                cls_f1 = (2 * cls_precision * cls_recall) / (cls_precision
+                                                             + cls_recall) if (cls_precision + cls_recall) > 0 else 0.0
+
+                precision_values.append(cls_precision)
+                recall_values.append(cls_recall)
+                f1_values.append(cls_f1)
+
+            # Macro average (simple average across all classes)
+            precision = sum(precision_values) / len(precision_values) if precision_values else 0.0
+            recall = sum(recall_values) / len(recall_values) if recall_values else 0.0
+            f1_score = sum(f1_values) / len(f1_values) if f1_values else 0.0
+
+            # Calculate ratios
+            yes_ratio = yes_count / total_count if total_count > 0 else 0.0
+            maybe_ratio = maybe_count / total_count if total_count > 0 else 0.0
+
+            return {
+                'accuracy': accuracy,
+                'precision': precision,
+                'recall': recall,
+                'f1_score': f1_score,
+                'yes_ratio': yes_ratio,
+                'maybe_ratio': maybe_ratio
+            }
+
+        overall_metrics = compute_metrics(sample_scores)
+        agg_scores = []
+        for metric_name, value in overall_metrics.items():
+            agg_scores.append(AggScore(metric_name=metric_name, score=value, num=len(sample_scores), metadata={}))
+
+        return agg_scores

evalscope/benchmarks/qasc/qasc_adapter.py

@@ -0,0 +1,35 @@
+from evalscope.api.benchmark import BenchmarkMeta, MultiChoiceAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.multi_choices import MultipleChoiceTemplate
+
+DESCRIPTION = (
+    'QASC is a question-answering dataset with a focus on sentence composition. '
+    'It consists of 9,980 8-way multiple-choice questions about grade school science.'
+)
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='qasc',
+        pretty_name='QASC',
+        tags=[Tags.KNOWLEDGE, Tags.MULTIPLE_CHOICE],
+        description=DESCRIPTION.strip(),
+        dataset_id='extraordinarylab/qasc',
+        metric_list=['acc'],
+        few_shot_num=0,
+        train_split=None,
+        eval_split='validation',
+        prompt_template=MultipleChoiceTemplate.SINGLE_ANSWER,
+    )
+)
+class QASCAdapter(MultiChoiceAdapter):
+
+    def record_to_sample(self, record) -> Sample:
+        return Sample(
+            input=record['question'],
+            choices=record['choices'],
+            target=record['answer'],
+            metadata={},
+        )

evalscope/benchmarks/race/race_adapter.py

@@ -1,136 +1,49 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 
-import os
-
-from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.constants import EvalType
-from evalscope.metrics import AverageAccuracy, exact_match
-from evalscope.models import MultiChoiceModelAdapter
-from evalscope.utils import ResponseParser
-from evalscope.utils.io_utils import jsonl_to_list
+from evalscope.api.benchmark import BenchmarkMeta, MultiChoiceAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
 from evalscope.utils.logger import get_logger
+from evalscope.utils.multi_choices import MultipleChoiceTemplate
 
 # flake8: noqa
 
 logger = get_logger()
 
 
-@Benchmark.register(
-    name='race',
-    dataset_id='modelscope/race',
-    model_adapter=MultiChoiceModelAdapter,
-    subset_list=['high', 'middle'],
-    metric_list=[AverageAccuracy],
-    few_shot_num=3,
-    train_split='train',
-    eval_split='test',
+@register_benchmark(
+    BenchmarkMeta(
+        name='race',
+        pretty_name='RACE',
+        tags=[Tags.REASONING, Tags.MULTIPLE_CHOICE],
+        description=
+        'RACE is a benchmark for testing reading comprehension and reasoning abilities of neural models. It is constructed from Chinese middle and high school examinations.',  # noqa: E501
+        dataset_id='evalscope/race',
+        metric_list=['acc'],
+        subset_list=['high', 'middle'],
+        few_shot_num=3,
+        train_split='train',
+        eval_split='test',
+        prompt_template=MultipleChoiceTemplate.SINGLE_ANSWER_COT,
+    )
 )
-class RACEAdapter(DataAdapter):
-
-    choices = ['A', 'B', 'C', 'D']
+class RACEAdapter(MultiChoiceAdapter):
 
     def __init__(self, **kwargs):
-        few_shot_num = kwargs.get('few_shot_num', 3)
-        if few_shot_num > 3:
-            logger.warning(f'few_shot_num <= 3 for RACE, but got {few_shot_num}. Use 3-shot by default.')
-            kwargs['few_shot_num'] = 3
-
         super().__init__(**kwargs)
 
-    def load_from_disk(self, dataset_name_or_path, subset_list, work_dir, **kwargs) -> dict:
-        data_dict = {}
-        for subset_name in subset_list:
-            data_dict[subset_name] = {}
-            for split in [self.train_split, self.eval_split]:
-                if os.path.exists(dataset_name_or_path):
-                    file_path = os.path.join(dataset_name_or_path, subset_name, f'{split}.jsonl')
-                else:
-                    file_path = os.path.join(work_dir, dataset_name_or_path, subset_name, f'{split}.jsonl')
-                if os.path.exists(file_path):
-                    data_dict[subset_name][split] = jsonl_to_list(file_path)
-
-        return data_dict
-
-    def gen_prompt(self, input_d: dict, subset_name: str, few_shot_list: list, **kwargs) -> dict:
-        """
-        Generate model prompt from raw input, unify the prompt format for RACE benchmark.
-
-        Args:
-            input_d (dict): The raw input. A single data format of the RACE:
-
-            {'example_id': 'high3680.txt',
-            'article': 'Astronauts on shorter shuttle missions often work very long days. Tasks are scheduled so tightly that break times are often used to finish the day's work. This type of schedule is far too demanding for long missions on the International Space Station(ISS). ISS crewmembers usually live in space for at least a quarter of a year. They work five days on and two days off to _ the normal way they do things on Earth as much as possible. Weekends give the crew valuable time to rest and do a few hours of housework. They can communicate with family and friends by email , internet phone and through private video conferences. While astronauts cannot go to a baseball game or a movie in orbit, there are many familiar activities that they can still enjoy . Before a mission, the family and friends of each ISS crewmember put together a collection of family photos, messages, videos and reading material for the astronauts to look at when they will be floating 370 kilometers above the Earth. During their mission, the crew also receives care packages with CDs, books, magazines, photos and letters . And as from early 2010, the internet became available on the ISS , giving astronauts the chance to do some "web surfing "in their personal time. Besides relaxing with these more common entertainments, astronauts can simply enjoy the experience of living in space. Many astronauts say that one of the most relaxing things to do in space is to look out the window and stare at the universe and the Earth's vast land mass and oceans.',
-            'answer': 'C',
-            'question': 'The passage mainly discusses how astronauts _ .',
-            'options': [
-                "work for longer missions in space",
-                "connect with people on the Earth",
-                "spend their free time in space",
-                "observe the Earth from space"]}
-
-        Returns:
-            {'data': [(context, continuation), ...]}
-
-        """
-        prompt = 'The following are multiple choice reading comprehension questions (with answers).\n\n'.format(
-            self._format_subject(subset_name))
-        few_shot_prompts = [self._generate_prompt(input_d=sample, include_answer=True) for sample in few_shot_list]
-
-        context: str = '\n'.join(few_shot_prompts) + '\n'
-        context += self._generate_prompt(input_d=input_d, include_answer=False)
-        context = prompt + context
-
-        full_prompt: str = context.strip() + self._generate_prompt(input_d=input_d, include_answer=False)
-
-        return {'data': [full_prompt], 'multi_choices': self.choices, 'system_prompt': self.prompt_template}
-
-    def get_gold_answer(self, input_d: dict) -> str:
-        # Get the gold choice
-        return input_d.get('answer', '')
-
-    def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = EvalType.CHECKPOINT) -> str:
-        """
-        Parse the model output to get the answer. Could be the best choice index.
-
-        Args:
-            result: Predicted answer from the model. Usually a string for chat.
-            raw_input_d: The raw input. Depending on the dataset.
-            eval_type: The evaluation type. e.g. 'checkpoint' or 'service' or 'custom'.
-
-        Returns:
-            The parsed answer. Depending on the dataset. Usually a string for chat.
-        """
-        if eval_type == EvalType.CHECKPOINT:
-            return result
-        elif eval_type == EvalType.SERVICE:
-            return ResponseParser.parse_first_option_with_choices(result, self.choices)  # TODO: to be checked !
-        elif eval_type == EvalType.CUSTOM:
-            return ResponseParser.parse_first_option_with_choices(result, self.choices)  # TODO: to be checked !
-        else:
-            raise ValueError(f'Unknown eval_type: {eval_type}')
-
-    def match(self, gold: str, pred: str) -> float:
-        return exact_match(gold=gold, pred=pred)
-
-    @classmethod
-    def _generate_prompt(cls, input_d: dict, include_answer=True) -> str:
-
-        input_choices: list = input_d['options']
-
-        example: str = 'Article:\n{}\nQuestion:\n{}'.format(input_d['article'], input_d['question'])
-        for j in range(len(cls.choices)):
-            example += '\n{}. {}'.format(cls.choices[j], input_choices[j])
-
-        example += '\nAnswer:'
-        if include_answer:
-            example += ' {}\n\n'.format(input_d['answer'])
+        if self.few_shot_num > 3:
+            logger.warning(f'few_shot_num <= 3 for RACE, but got {self.few_shot_num}. Use 3-shot by default.')
+            self.few_shot_num = 3
 
-        return example
+    def record_to_sample(self, record) -> Sample:
+        # Format the article and question as context
+        context = f"Article:\n{record['article']}\nQuestion:\n{record['question']}"
 
-    @classmethod
-    def _format_subject(cls, subject):
-        l = subject.split('_')
-        s = ''
-        for entry in l:
-            s += ' ' + entry
-        return s
+        return Sample(
+            input=context,
+            choices=record['options'],
+            target=record['answer'],
+            metadata={'example_id': record.get('example_id', 'unknown')},
+        )

evalscope/benchmarks/real_world_qa/real_world_qa_adapter.py

@@ -0,0 +1,64 @@
+import re
+from typing import Any, Dict, List
+
+from evalscope.api.benchmark import BenchmarkMeta, VisionLanguageAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.evaluator import TaskState
+from evalscope.api.messages import ChatMessageUser, Content, ContentImage, ContentText
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.io_utils import bytes_to_base64
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+SUBSET_LIST = ['default']
+
+OPEN_PROMPT = (
+    'Read the picture and solve the following problem step by step.'
+    'The last line of your response should be of the form'
+    ' "ANSWER: $ANSWER" (without quotes) where $ANSWER is the answer to the problem.\n\n'
+    '{question}\n\n'
+    'Remember to put your answer on its own line at the end in the form'
+    ' "ANSWER: $ANSWER" (without quotes) where $ANSWER is the answer to the problem,'
+    ' and you do not need to use a \\boxed command.'
+)
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='real_world_qa',
+        pretty_name='RealWorldQA',
+        tags=[Tags.MULTI_MODAL, Tags.KNOWLEDGE, Tags.QA],
+        description=
+        'RealWorldQA is a benchmark designed to evaluate the real-world spatial understanding capabilities of multimodal AI models, contributed by XAI. It assesses how well these models comprehend physical environments. The benchmark consists of 700+ images, each accompanied by a question and a verifiable answer. These images are drawn from real-world scenarios, including those captured from vehicles. The goal is to advance AI models\' understanding of our physical world.',  # noqa: E501
+        dataset_id='lmms-lab/RealWorldQA',
+        subset_list=SUBSET_LIST,
+        metric_list=['acc'],
+        eval_split='test',
+        prompt_template=OPEN_PROMPT,
+    )
+)
+class RealWorldQAAdapter(VisionLanguageAdapter):
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        content_list: list[Content] = [ContentText(text=OPEN_PROMPT.format(question=record['question']))]
+        image = record.get('image')
+        if image:
+            image_base64 = bytes_to_base64(image['bytes'], format='webp', add_header=True)
+            content_list.append(ContentImage(image=image_base64))
+        return Sample(
+            input=[ChatMessageUser(content=content_list)],
+            target=record['answer'],
+            metadata={'image_path': record['image_path']}
+        )
+
+    def extract_answer(self, prediction: str, task_state: TaskState) -> str:
+        pattern = r'ANSWER:\s*(.*)'
+        match = re.search(pattern, prediction)
+        if match:
+            return match.group(1).strip()
+        return ''

evalscope/benchmarks/sciq/sciq_adapter.py

@@ -0,0 +1,36 @@
+from evalscope.api.benchmark import BenchmarkMeta, MultiChoiceAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.multi_choices import MultipleChoiceTemplate
+
+DESCRIPTION = (
+    'The SciQ dataset contains crowdsourced science exam questions about Physics, '
+    'Chemistry and Biology, among others. For the majority of the questions, '
+    'an additional paragraph with supporting evidence for the correct answer is provided.'
+)  # noqa: E501
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='sciq',
+        pretty_name='SciQ',
+        tags=[Tags.READING_COMPREHENSION, Tags.KNOWLEDGE, Tags.MULTIPLE_CHOICE],
+        description=DESCRIPTION.strip(),
+        dataset_id='extraordinarylab/sciq',
+        metric_list=['acc'],
+        few_shot_num=0,
+        train_split=None,
+        eval_split='test',
+        prompt_template=MultipleChoiceTemplate.SINGLE_ANSWER,
+    )
+)
+class SciQAdapter(MultiChoiceAdapter):
+
+    def record_to_sample(self, record) -> Sample:
+        return Sample(
+            input=record['question'],
+            choices=record['choices'],
+            target=record['answer'],
+            metadata={},
+        )

evalscope/benchmarks/seed_bench_2_plus/seed_bench_2_plus_adapter.py

@@ -0,0 +1,72 @@
+# flake8: noqa: E501
+import re
+from typing import Any, Dict, List
+
+from evalscope.api.benchmark import BenchmarkMeta, VisionLanguageAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.evaluator import TaskState
+from evalscope.api.messages import ChatMessageUser, Content, ContentImage, ContentText
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.io_utils import bytes_to_base64
+from evalscope.utils.logger import get_logger
+from evalscope.utils.multi_choices import MultipleChoiceTemplate, parse_answers, prompt
+
+logger = get_logger()
+
+MULT_CHOICE_PROMPT = MultipleChoiceTemplate.SINGLE_ANSWER_COT
+
+SUBSET_LIST = ['chart', 'web', 'map']
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='seed_bench_2_plus',
+        pretty_name='SEED-Bench-2-Plus',
+        dataset_id='evalscope/SEED-Bench-2-Plus',
+        tags=[Tags.KNOWLEDGE, Tags.REASONING, Tags.MULTIPLE_CHOICE, Tags.MULTI_MODAL],
+        description=
+        'SEED-Bench-2-Plus is a large-scale benchmark to evaluate Multimodal Large Language Models (MLLMs). It consists of 2.3K multiple-choice questions with precise human annotations, spanning three broad categories: Charts, Maps, and Webs, each of which covers a wide spectrum of text-rich scenarios in the real world.',
+        subset_list=SUBSET_LIST,
+        metric_list=['acc'],
+        eval_split='test',
+        prompt_template=MULT_CHOICE_PROMPT,
+    )
+)
+class SeedBench2PlusAdapter(VisionLanguageAdapter):
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.reformat_subset = True
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        question = record['question']
+        answers_list = [record['choice_A'], record['choice_B'], record['choice_C'], record['choice_D']]
+        input_text = prompt(question=question, choices=answers_list, template=self.prompt_template)
+        content_list: List[Content] = [ContentText(text=input_text)]
+        image = record['image']
+        if image:
+            image_base64 = bytes_to_base64(image['bytes'], format='png', add_header=True)
+            content_list.append(ContentImage(image=image_base64))
+        metadata = {
+            'data_id': record['data_id'],
+            'question_id': record['question_id'],
+            'question_image_subtype': record['question_image_subtype'],
+            'data_source': record['data_source'],
+            'data_type': record['data_type'],
+            'level': record['level'],
+            'subpart': record['subpart'],
+            'version': record['version'],
+        }
+        label_answer = record['answer']
+        return Sample(
+            input=[ChatMessageUser(content=content_list)],
+            choices=answers_list,
+            target=label_answer,
+            subset_key=record['question_image_type'],
+            metadata=metadata,
+        )
+
+    def extract_answer(self, prediction: str, task_state: TaskState) -> str:
+        answers = parse_answers(task_state)
+        return ''.join(sorted(list(answers)))