evalscope 0.17.1__py3-none-any.whl → 1.0.0__py3-none-any.whl

evalscope/benchmarks/aime/aime24_adapter.py

@@ -1,5 +1,12 @@
-from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.metrics import extract_answer, math_equal, strip_answer_string
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
+from typing import Any, Dict
+
+from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.evaluator import TaskState
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
 from evalscope.utils.logger import get_logger
 
 # flake8: noqa
@@ -7,46 +14,37 @@ from evalscope.utils.logger import get_logger
 logger = get_logger()
 
 
-@Benchmark.register(
-    name='aime24',
-    pretty_name='AIME-2024',
-    tags=['Mathematics'],
-    description=
-    'The AIME 2024 benchmark is based on problems from the American Invitational Mathematics Examination, a prestigious high school mathematics competition. This benchmark tests a model’s ability to solve challenging mathematics problems by generating step-by-step solutions and providing the correct final answer.',  # noqa: E501
-    dataset_id='HuggingFaceH4/aime_2024',
-    subset_list=['default'],
-    metric_list=['AveragePass@1'],
-    few_shot_num=0,
-    train_split=None,
-    eval_split='train',  # Only train set is available
-    prompt_template='{query}\nPlease reason step by step, and put your final answer within \\boxed{{}}.',
+@register_benchmark(
+    BenchmarkMeta(
+        name='aime24',
+        pretty_name='AIME-2024',
+        tags=[Tags.MATH, Tags.REASONING],
+        description=
+        'The AIME 2024 benchmark is based on problems from the American Invitational Mathematics Examination, a prestigious high school mathematics competition. This benchmark tests a model\'s ability to solve challenging mathematics problems by generating step-by-step solutions and providing the correct final answer.',  # noqa: E501
+        dataset_id='HuggingFaceH4/aime_2024',
+        subset_list=['default'],
+        metric_list=[{
+            'acc': {
+                'numeric': True
+            }
+        }],
+        few_shot_num=0,
+        train_split=None,
+        eval_split='train',  # Only train set is available
+        prompt_template='{question}\nPlease reason step by step, and put your final answer within \\boxed{{}}.',
+    )
 )
-class AIME24Adapter(DataAdapter):
+class AIME24Adapter(DefaultDataAdapter):
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
 
-    def gen_prompt(self, input_d: dict, few_shot_list: list, **kwargs) -> dict:
-        """
-        Generate the prompt for the model input.
-        """
-        problem = input_d['problem']
-        full_prompt = self.prompt_template.format(query=problem)
-
-        return self.gen_prompt_data(full_prompt)
-
-    def get_gold_answer(self, input_d: dict) -> str:
-        # Extract the gold answer from the input dict.
-        return strip_answer_string(input_d['answer'])
-
-    def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = 'checkpoint') -> str:
-        """
-        Parse the model output to get the answer. Could be the best choice index.
-        """
-        # Note: Use same extraction method for both of checkpoint/service/custom
-        result = strip_answer_string(extract_answer(result))
-        return result
-
-    def match(self, gold: str, pred: str) -> float:
-        res = math_equal(pred, gold)
-        return 1.0 if res else 0.0
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        return Sample(
+            input=record['problem'],
+            target=record['answer'],
+            metadata={
+                'problem_id': record.get('id', ''),
+                'solution': record.get('solution', ''),
+            },
+        )

evalscope/benchmarks/aime/aime25_adapter.py

@@ -1,5 +1,12 @@
-from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.metrics import extract_answer, math_equal, strip_answer_string
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
+from typing import Any, Dict
+
+from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.evaluator import TaskState
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
 from evalscope.utils.logger import get_logger
 
 # flake8: noqa
@@ -7,46 +14,33 @@ from evalscope.utils.logger import get_logger
 logger = get_logger()
 
 
-@Benchmark.register(
-    name='aime25',
-    pretty_name='AIME-2025',
-    tags=['Mathematics'],
-    description=
-    'The AIME 2025 benchmark is based on problems from the American Invitational Mathematics Examination, a prestigious high school mathematics competition. This benchmark tests a model’s ability to solve challenging mathematics problems by generating step-by-step solutions and providing the correct final answer.',
-    dataset_id='opencompass/AIME2025',
-    subset_list=['AIME2025-I', 'AIME2025-II'],
-    metric_list=['AveragePass@1'],
-    few_shot_num=0,
-    train_split=None,
-    eval_split='test',  # Only train set is available
-    prompt_template='{query}\nPlease reason step by step, and put your final answer within \\boxed{{}}.',
+@register_benchmark(
+    BenchmarkMeta(
+        name='aime25',
+        pretty_name='AIME-2025',
+        tags=[Tags.MATH, Tags.REASONING],
+        description=
+        'The AIME 2025 benchmark is based on problems from the American Invitational Mathematics Examination, a prestigious high school mathematics competition. This benchmark tests a model\'s ability to solve challenging mathematics problems by generating step-by-step solutions and providing the correct final answer.',
+        dataset_id='opencompass/AIME2025',
+        subset_list=['AIME2025-I', 'AIME2025-II'],
+        metric_list=[{
+            'acc': {
+                'numeric': True
+            }
+        }],
+        few_shot_num=0,
+        train_split=None,
+        eval_split='test',
+        prompt_template='{question}\nPlease reason step by step, and put your final answer within \\boxed{{}}.',
+    )
 )
-class AIME25Adapter(DataAdapter):
+class AIME25Adapter(DefaultDataAdapter):
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
 
-    def gen_prompt(self, input_d: dict, few_shot_list: list, **kwargs) -> dict:
-        """
-        Generate the prompt for the model input.
-        """
-        problem = input_d['question']
-        full_prompt = self.prompt_template.format(query=problem)
-
-        return self.gen_prompt_data(full_prompt)
-
-    def get_gold_answer(self, input_d: dict) -> str:
-        # Extract the gold answer from the input dict.
-        return strip_answer_string(input_d['answer'])
-
-    def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = 'checkpoint') -> str:
-        """
-        Parse the model output to get the answer. Could be the best choice index.
-        """
-        # Note: Use same extraction method for both of checkpoint/service/custom
-        result = strip_answer_string(extract_answer(result))
-        return result
-
-    def match(self, gold: str, pred: str) -> float:
-        res = math_equal(pred, gold)
-        return 1.0 if res else 0.0
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        return Sample(
+            input=record['question'],
+            target=record['answer'],
+        )

evalscope/benchmarks/alpaca_eval/alpaca_eval_adapter.py

@@ -1,16 +1,17 @@
 import re
-from collections import defaultdict
-from typing import Any, List
-
-from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.metrics import LLMJudge, Metric, mean, metric_registry
+from typing import Any, Dict
+
+from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.evaluator import TaskState
+from evalscope.api.metric import Score
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
 from evalscope.utils.logger import get_logger
 
-# flake8: noqa
-
 logger = get_logger()
 
-GRADER_SYSTEM_PROMPT = """You are a highly efficient assistant, who evaluates and selects the best large language model (LLMs) based on the quality of their responses to a given instruction. This process will be used to create a leaderboard reflecting the most accurate and human-preferred answers."""
+GRADER_SYSTEM_PROMPT = """You are a highly efficient assistant, who evaluates and selects the best large language model (LLMs) based on the quality of their responses to a given instruction. This process will be used to create a leaderboard reflecting the most accurate and human-preferred answers."""  # noqa: E501
 
 GRADER_TEMPLATE = """
 I require a leaderboard for various large language models. I'll provide you with prompts given to these models and their corresponding outputs. Your task is to assess these responses, and select the model that produces the best output from a human perspective.
@@ -44,64 +45,89 @@ Evaluate the models based on the quality and relevance of their outputs, and sel
 """.strip()  # noqa: E501
 
 
-@Benchmark.register(
-    name='alpaca_eval',
-    pretty_name='AlpacaEval2.0',
-    tags=['Instruction-Following', 'Arena'],
-    description='Alpaca Eval 2.0 is an enhanced framework for evaluating instruction-following language models, '
-    'featuring an improved auto-annotator, updated baselines, and continuous preference calculation to '
-    'provide more accurate and cost-effective model assessments. '
-    'Currently not support `length-controlled winrate`; the official Judge model is `gpt-4-1106-preview`, while the baseline model is `gpt-4-turbo`.',  # noqa: E501
-    dataset_id='AI-ModelScope/alpaca_eval',
-    subset_list=['alpaca_eval_gpt4_baseline'],
-    metric_list=['winrate'],
-    few_shot_num=0,
-    train_split=None,
-    eval_split='eval')
-class AlpacaEvalAdapter(DataAdapter):
+@register_benchmark(
+    BenchmarkMeta(
+        name='alpaca_eval',
+        pretty_name='AlpacaEval2.0',
+        tags=[Tags.INSTRUCTION_FOLLOWING, Tags.ARENA],
+        description='Alpaca Eval 2.0 is an enhanced framework for evaluating instruction-following language models, '
+        'featuring an improved auto-annotator, updated baselines, and continuous preference calculation to '
+        'provide more accurate and cost-effective model assessments. '
+        'Currently not support `length-controlled winrate`; the official Judge model is `gpt-4-1106-preview`, while the baseline model is `gpt-4-turbo`.',  # noqa: E501
+        dataset_id='AI-ModelScope/alpaca_eval',
+        subset_list=['alpaca_eval_gpt4_baseline'],
+        metric_list=['winrate'],
+        few_shot_num=0,
+        train_split=None,
+        eval_split='eval',
+        prompt_template='{question}'
+    )
+)
+class AlpacaEvalAdapter(DefaultDataAdapter):
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
 
-        # register metrics
-        metric_registry.register(Metric(name='winrate', object=mean))
-
-        # whether to use LLM as a judge
-        self.llm_as_a_judge = True
-
-    def gen_prompt(self, input_d: dict, subset_name: str, few_shot_list: list, **kwargs) -> dict:
-        question = input_d['instruction']
-        return self.gen_prompt_data(question)
-
-    def get_gold_answer(self, input_d: dict) -> str:
-        return input_d['output']
+        self._use_llm_judge = True  # Use LLM as a judge by default
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        """
+        Convert a data record to a Sample object.
+
+        Args:
+            record (Dict[str, Any]): Input data record.
+
+        Returns:
+            Sample: Sample object with input, target, and metadata.
+        """
+        instruction = record['instruction']
+        baseline_output = record['output']  # baseline model output
+
+        return Sample(
+            input=instruction,
+            target=baseline_output,
+            metadata={
+                'generator': record.get('generator', 'unknown'),
+                'dataset': record.get('dataset', 'unknown')
+            }
+        )
+
+    def llm_match_score(
+        self,
+        original_prediction: str,
+        filtered_prediction: str,
+        reference: str,
+        task_state: TaskState,
+    ) -> Score:
+        score = Score(
+            extracted_prediction=filtered_prediction,
+            prediction=original_prediction,
+        )
+
+        instruction = task_state.input_text
+
+        # Request judge and obtain score
+        # reference is baseline answer 'm', filtered_prediction is model answer 'M'
+        prompt = GRADER_TEMPLATE.format(instruction=instruction, output_1=reference, output_2=filtered_prediction)
+        judge_response = self.llm_judge.judge(prompt, system_prompt=GRADER_SYSTEM_PROMPT)
 
-    def parse_pred_result(self, result: str, raw_input_d: dict = None, **kwargs) -> str:
-        return result.strip()
-
-    def match(self, gold: str, pred: str):
-        # simple match
-        logger.warning(f'Please use LLMJudge to match the result for {self.name}')
-        return None
-
-    def llm_match(self, gold: Any, pred: Any, judge: LLMJudge, **kwargs) -> bool:
-        raw_input = kwargs.get('raw_input', None)
-        instruction = raw_input['instruction']
-        # gold is baseline answer 'm', pred is model answer 'M'
-        prompt = GRADER_TEMPLATE.format(instruction=instruction, output_1=gold, output_2=pred)
-        # get grading response
-        grading_response = judge(prompt, system_prompt=GRADER_SYSTEM_PROMPT)
         # parse grading response
-        match = re.search(r'(m|M)', grading_response)
+        match = re.search(r'(m|M)', judge_response)
         res = match.group(0) if match else None
+
         if res:
-            return res == 'M'
+            winrate = 1 if res == 'M' else 0
         else:
-            logger.info(f'Failed to parse grading response: {prompt=}\n {grading_response=}')
-            return None
-
-    def compute_metric(self, review_res_list: List[bool], **kwargs) -> List[dict]:
-        # zip dict answers
-        res_list = [res for res in review_res_list if res is not None]
-
-        return super().compute_metric(res_list, **kwargs)
+            logger.info(f'Failed to parse grading response: {prompt=}\n {judge_response=}')
+            winrate = 0
+
+        # Set score based on the match result
+        score.value = {'winrate': winrate}
+        score.explanation = f'LLM judge: {judge_response}'
+        score.metadata = {
+            'source': 'llm_judge',
+            'judge_strategy': self.judge_strategy,
+            'model': self.llm_judge.model_id
+        }
+        score.main_score_name = 'winrate'
+        return score

evalscope/benchmarks/arc/arc_adapter.py

@@ -1,159 +1,46 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 
-import json
-import os
-
-from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.constants import EvalType, OutputType
-from evalscope.metrics import exact_match
-from evalscope.metrics.completion_parsers import ResponseParser
+from evalscope.api.benchmark import BenchmarkMeta, MultiChoiceAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
 from evalscope.utils.logger import get_logger
-
-# flake8: noqa
+from evalscope.utils.multi_choices import MultipleChoiceTemplate
 
 logger = get_logger()
 
 
-@Benchmark.register(
-    name='arc',
-    pretty_name='ARC',
-    tags=['Reasoning', 'MCQ'],
-    description=
-    'The ARC (AI2 Reasoning Challenge) benchmark is designed to evaluate the reasoning capabilities of AI models through multiple-choice questions derived from science exams. It includes two subsets: ARC-Easy and ARC-Challenge, which vary in difficulty.',  # noqa: E501
-    dataset_id='modelscope/ai2_arc',
-    model_adapter=OutputType.GENERATION,
-    output_types=[OutputType.MULTIPLE_CHOICE, OutputType.GENERATION],
-    subset_list=['ARC-Easy', 'ARC-Challenge'],
-    metric_list=['AverageAccuracy'],
-    few_shot_num=0,
-    train_split='train',
-    eval_split='test',
-    prompt_template=
-    'Given the following question and four candidate answers (A, B, C and D), choose the best answer.\n{query}\nYour response should end with "The best answer is [the_answer_letter]" where the [the_answer_letter] is one of A, B, C or D.',  # noqa
+@register_benchmark(
+    BenchmarkMeta(
+        name='arc',
+        pretty_name='ARC',
+        tags=[Tags.REASONING, Tags.MULTIPLE_CHOICE],
+        description=
+        'The ARC (AI2 Reasoning Challenge) benchmark is designed to evaluate the reasoning capabilities of AI models through multiple-choice questions derived from science exams. It includes two subsets: ARC-Easy and ARC-Challenge, which vary in difficulty.',  # noqa: E501
+        dataset_id='allenai/ai2_arc',
+        subset_list=['ARC-Easy', 'ARC-Challenge'],
+        metric_list=['acc'],
+        few_shot_num=0,
+        train_split='train',
+        eval_split='test',
+        prompt_template=MultipleChoiceTemplate.SINGLE_ANSWER,
+    )
 )
-class ARCAdapter(DataAdapter):
+class ARCAdapter(MultiChoiceAdapter):
 
     def __init__(self, **kwargs):
-        few_shot_num = kwargs.get('few_shot_num', None)
-        if few_shot_num is None:
-            # Use 0-shot by default
-            logger.info(f'Set 0-shot examples by system for ARC.')
-            few_shot_num = 0
-
-        if few_shot_num != 0:
-            logger.warning(f'few_shot_num is recommended to set 0 for ARC, got {few_shot_num}.')
-
         super().__init__(**kwargs)
 
-        self.choices = ['A', 'B', 'C', 'D']
-
-    def load_from_disk(self, dataset_name_or_path, subset_list, work_dir, **kwargs) -> dict:
-        """
-        Load the dataset from local disk.
-
-        dataset_name_or_path: str, the dataset id or path. e.g. 'arc'
-        subset_list: list, the subset list to load. e.g. ['ARC-Easy', 'ARC-Challenge']
-        work_dir: str, the local root data directory. e.g. '/path/to/data'
-        kwargs: dict, other arguments.
-        """
-        data_dict = {}
-        for subset_name in subset_list:
-            if os.path.exists(dataset_name_or_path):
-                subset_path = os.path.join(dataset_name_or_path, subset_name)
-            else:
-                subset_path = os.path.join(work_dir, dataset_name_or_path, subset_name)
-            for split_name in ['Train', 'Test']:
-                split_path = os.path.join(subset_path, f'{subset_name}-{split_name}.jsonl')
-                if os.path.exists(split_path):
-                    with open(split_path, 'r', errors='ignore', encoding='utf-8') as in_f:
-                        rows = []
-                        for line in in_f:
-                            item = json.loads(line.strip())
-                            raw_choices = item['question']['choices']
-                            rows.append({
-                                'id': item['id'],
-                                'question': item['question']['stem'],
-                                'choices': {
-                                    'text': [d['text'] for d in raw_choices],
-                                    'label': [d['label'] for d in raw_choices]
-                                },
-                                'answerKey': item['answerKey'],
-                            })
-
-                        if subset_name in data_dict:
-                            data_dict[subset_name].update({split_name.lower(): rows})
-                        else:
-                            data_dict[subset_name] = {split_name.lower(): rows}
-
-        return data_dict
-
-    def gen_prompt(self, input_d: dict, few_shot_list: list, **kwargs) -> dict:
-        """
-        Generate model prompt from raw data, unify the prompt format for ARC benchmark.
-
-        Args:
-            input_d (dict): The raw input. A single data format of the ARC:
-
-            {
-                'id': 'Mercury_7220990',
-                'question': 'Which factor will most likely cause a person to develop a fever?',
-                'choices':
-                    {
-                        'text':['a leg muscle relaxing after exercise',
-                                'a bacterial population in the bloodstream',
-                                'several viral particles on the skin',
-                                'carbohydrates being digested in the stomach'],
-                        'label': ['A', 'B', 'C', 'D']
-                    },
-                'answerKey': 'B'
-            }
-
-        Returns:
-            {'data': ['xxx'], 'multi_choices': ['A', 'B', 'C', 'D']}
-        """
-        few_shot_prompts = [self._generate_prompt(input_d=sample, include_answer=True) for sample in few_shot_list]
-        context = '\n'.join(few_shot_prompts) + self._generate_prompt(input_d=input_d, include_answer=False)
-
-        full_prompt = self.prompt_template.format(query=context)
-
-        return self.gen_prompt_data(full_prompt)
-
-    def get_gold_answer(self, input_d: dict) -> str:
-        # Get the gold choice
-        return input_d.get('answerKey', '')
-
-    def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = EvalType.CHECKPOINT) -> str:
-        """
-        Parse the model output to get the answer. Could be the best choice index.
-
-        Args:
-            result: Predicted answer from the model. Usually a string for chat.
-            raw_input_d (dict): The raw input. Depending on the dataset.
-            eval_type: 'checkpoint' or 'service' or `custom`, default: 'checkpoint'
-
-        Returns:
-            The parsed answer. Depending on the dataset. Usually a string for chat.
-        """
-        if self.model_adapter == OutputType.MULTIPLE_CHOICE:
-            return result
-        else:
-            return ResponseParser.parse_first_option(text=result, options=self.choices)
-
-    def match(self, gold: str, pred: str) -> float:
-        return exact_match(gold=gold, pred=pred)
-
-    @classmethod
-    def _generate_prompt(cls, input_d: dict, include_answer=True) -> str:
-
-        example: str = input_d['question']
-
-        choices_texts: list = input_d['choices']['text']
-        choices_labels: list = input_d['choices']['label']
-        choices_prompts: str = '\n'.join([label + '. ' + text for text, label in zip(choices_texts, choices_labels)])
-        example += '\n' + choices_prompts
-
-        if include_answer:
-            example += '\nAnswer:'
-            example += ' {}\n\n'.format(input_d['answerKey'])
-
-        return example
+    def record_to_sample(self, record) -> Sample:
+        # Convert choice labels to indices (A->0, B->1, etc.)
+        choice_texts = record['choices']['text']
+        answer_key = record['answerKey']
+
+        return Sample(
+            input=record['question'],
+            choices=choice_texts,
+            target=answer_key,
+            metadata={
+                'id': record.get('id', ''),
+            },
+        )