evalscope 0.10.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

evalscope/{metrics/math_accuracy.py → benchmarks/aime/math_normalize.py}

@@ -1,69 +1,53 @@
-# Copyright (c) Alibaba, Inc. and its affiliates.
+"""
+This logic is largely copied from the Hendrycks' MATH release (math_equivalence).
 
 
-# Adapted from https://github.com/EleutherAI/lm-evaluation-harness/blob/master/lm_eval/tasks/hendrycks_math.py
-def is_equiv(str1, str2, verbose=False):
-    if str1 is None and str2 is None:
-        print('WARNING: Both None')
-        return True
-    if str1 is None or str2 is None:
-        return False
+This file is adapted from OpenAI's PRM800K repository:
+https://github.com/openai/prm800k/blob/main/prm800k/grading/math_normalize.py
 
-    try:
-        ss1 = strip_string(str1)
-        ss2 = strip_string(str2)
-        if verbose:
-            print(ss1, ss2)
-        return ss1 == ss2
-    except Exception:
-        return str1 == str2
-
-
-def remove_boxed(s):
-    if '\\boxed ' in s:
-        left = '\\boxed '
-        assert s[:len(left)] == left
-        return s[len(left):]
-
-    left = '\\boxed{'
-
-    assert s[:len(left)] == left
-    assert s[-1] == '}'
-
-    return s[len(left):-1]
-
-
-def last_boxed_only_string(string):
-    idx = string.rfind('\\boxed')
-    if '\\boxed ' in string:
-        return '\\boxed ' + string.split('\\boxed ')[-1].split('$')[0]
-    if idx < 0:
-        idx = string.rfind('\\fbox')
-        if idx < 0:
-            return None
-
-    i = idx
-    right_brace_idx = None
-    num_left_braces_open = 0
-    while i < len(string):
-        if string[i] == '{':
-            num_left_braces_open += 1
-        if string[i] == '}':
-            num_left_braces_open -= 1
-            if num_left_braces_open == 0:
-                right_brace_idx = i
-                break
-        i += 1
-
-    if right_brace_idx is None:
-        retval = None
-    else:
-        retval = string[idx:right_brace_idx + 1]
+Original License:
+MIT License
+
+Copyright (c) 2023 OpenAI
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
 
-    return retval
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+"""
+# flake8: noqa
+import re
+from typing import Optional
+
+
+def normalize_answer(answer: Optional[str]) -> Optional[str]:
+    if answer is None:
+        return None
+    answer = answer.strip()
+    try:
+        # Remove enclosing `\text{}`.
+        m = re.search('^\\\\text\{(?P<text>.+?)\}$', answer)
+        if m is not None:
+            answer = m.group('text').strip()
+        return _strip_string(answer)
+    except:
+        return answer
 
 
-def fix_fracs(string):
+def _fix_fracs(string):
     substrs = string.split('\\frac')
     new_str = substrs[0]
     if len(substrs) > 1:
@@ -75,7 +59,7 @@ def fix_fracs(string):
             else:
                 try:
                     assert len(substr) >= 2
-                except AssertionError:
+                except:
                     return string
                 a = substr[0]
                 b = substr[1]
@@ -95,7 +79,7 @@ def fix_fracs(string):
     return string
 
 
-def fix_a_slash_b(string):
+def _fix_a_slash_b(string):
     if len(string.split('/')) != 2:
         return string
     a = string.split('/')[0]
@@ -106,11 +90,11 @@ def fix_a_slash_b(string):
         assert string == '{}/{}'.format(a, b)
         new_string = '\\frac{' + str(a) + '}{' + str(b) + '}'
         return new_string
-    except AssertionError:
+    except:
         return string
 
 
-def remove_right_units(string):
+def _remove_right_units(string):
     # "\\text{ " only ever occurs (at least in the val set) when describing units
     if '\\text{ ' in string:
         splits = string.split('\\text{ ')
@@ -120,7 +104,7 @@ def remove_right_units(string):
         return string
 
 
-def fix_sqrt(string):
+def _fix_sqrt(string):
     if '\\sqrt' not in string:
         return string
     splits = string.split('\\sqrt')
@@ -135,23 +119,28 @@ def fix_sqrt(string):
     return new_string
 
 
-def strip_string(string):
+def _strip_string(string):
     # linebreaks
     string = string.replace('\n', '')
+    # print(string)
 
     # remove inverse spaces
     string = string.replace('\\!', '')
+    # print(string)
 
     # replace \\ with \
     string = string.replace('\\\\', '\\')
+    # print(string)
 
     # replace tfrac and dfrac with frac
     string = string.replace('tfrac', 'frac')
     string = string.replace('dfrac', 'frac')
+    # print(string)
 
     # remove \left and \right
     string = string.replace('\\left', '')
     string = string.replace('\\right', '')
+    # print(string)
 
     # Remove circ (degrees)
     string = string.replace('^{\\circ}', '')
@@ -161,11 +150,11 @@ def strip_string(string):
     string = string.replace('\\$', '')
 
     # remove units (on the right)
-    string = remove_right_units(string)
+    string = _remove_right_units(string)
 
     # remove percentage
     string = string.replace('\\%', '')
-    string = string.replace('\%', '')  # noqa: W605
+    string = string.replace('\%', '')
 
     # " 0." equivalent to " ." and "{0." equivalent to "{." Alternatively, add "0" if "." is the start of the string
     string = string.replace(' .', ' 0.')
@@ -182,19 +171,19 @@ def strip_string(string):
             string = string.split('=')[1]
 
     # fix sqrt3 --> sqrt{3}
-    string = fix_sqrt(string)
+    string = _fix_sqrt(string)
 
     # remove spaces
     string = string.replace(' ', '')
 
-    # \frac1b or \frac12 --> \frac{1}{b} and \frac{1}{2}, etc. Even works with \frac1{72} (but not \frac{72}1). Also does a/b --> \\frac{a}{b}  # noqa: E501
-    string = fix_fracs(string)
+    # \frac1b or \frac12 --> \frac{1}{b} and \frac{1}{2}, etc. Even works with \frac1{72} (but not \frac{72}1). Also does a/b --> \\frac{a}{b}
+    string = _fix_fracs(string)
 
     # manually change 0.5 --> \frac{1}{2}
     if string == '0.5':
         string = '\\frac{1}{2}'
 
     # NOTE: X/Y changed to \frac{X}{Y} in dataset, but in simple cases fix in case the model output is X/Y
-    string = fix_a_slash_b(string)
+    string = _fix_a_slash_b(string)
 
     return string

evalscope/benchmarks/alpaca_eval/alpaca_eval_adapter.py

@@ -0,0 +1,133 @@
+import re
+from typing import Any, Dict
+
+from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.evaluator import TaskState
+from evalscope.api.metric import Score
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+GRADER_SYSTEM_PROMPT = """You are a highly efficient assistant, who evaluates and selects the best large language model (LLMs) based on the quality of their responses to a given instruction. This process will be used to create a leaderboard reflecting the most accurate and human-preferred answers."""  # noqa: E501
+
+GRADER_TEMPLATE = """
+I require a leaderboard for various large language models. I'll provide you with prompts given to these models and their corresponding outputs. Your task is to assess these responses, and select the model that produces the best output from a human perspective.
+
+## Instruction
+
+{{
+    "instruction": "{instruction}"
+}}
+
+## Model Outputs
+
+Here are the unordered outputs from the models. Each output is associated with a specific model, identified by a unique model identifier.
+
+{{
+    {{
+        "model_identifier": "m",
+        "output": "{output_1}"
+    }},
+    {{
+        "model_identifier": "M",
+        "output": "{output_2}"
+    }}
+}}
+
+## Task
+
+Evaluate the models based on the quality and relevance of their outputs, and select the model that generated the best output. Answer by providing the model identifier of the best model. We will use your output as the name of the best model, so make sure your output only contains one of the following model identifiers and nothing else (no quotes, no spaces, no new lines, ...): m or M.
+
+## Best Model Identifier
+""".strip()  # noqa: E501
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='alpaca_eval',
+        pretty_name='AlpacaEval2.0',
+        tags=[Tags.INSTRUCTION_FOLLOWING, Tags.ARENA],
+        description='Alpaca Eval 2.0 is an enhanced framework for evaluating instruction-following language models, '
+        'featuring an improved auto-annotator, updated baselines, and continuous preference calculation to '
+        'provide more accurate and cost-effective model assessments. '
+        'Currently not support `length-controlled winrate`; the official Judge model is `gpt-4-1106-preview`, while the baseline model is `gpt-4-turbo`.',  # noqa: E501
+        dataset_id='AI-ModelScope/alpaca_eval',
+        subset_list=['alpaca_eval_gpt4_baseline'],
+        metric_list=['winrate'],
+        few_shot_num=0,
+        train_split=None,
+        eval_split='eval',
+        prompt_template='{question}'
+    )
+)
+class AlpacaEvalAdapter(DefaultDataAdapter):
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        self._use_llm_judge = True  # Use LLM as a judge by default
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        """
+        Convert a data record to a Sample object.
+
+        Args:
+            record (Dict[str, Any]): Input data record.
+
+        Returns:
+            Sample: Sample object with input, target, and metadata.
+        """
+        instruction = record['instruction']
+        baseline_output = record['output']  # baseline model output
+
+        return Sample(
+            input=instruction,
+            target=baseline_output,
+            metadata={
+                'generator': record.get('generator', 'unknown'),
+                'dataset': record.get('dataset', 'unknown')
+            }
+        )
+
+    def llm_match_score(
+        self,
+        original_prediction: str,
+        filtered_prediction: str,
+        reference: str,
+        task_state: TaskState,
+    ) -> Score:
+        score = Score(
+            extracted_prediction=filtered_prediction,
+            prediction=original_prediction,
+        )
+
+        instruction = task_state.input_text
+
+        # Request judge and obtain score
+        # reference is baseline answer 'm', filtered_prediction is model answer 'M'
+        prompt = GRADER_TEMPLATE.format(instruction=instruction, output_1=reference, output_2=filtered_prediction)
+        judge_response = self.llm_judge.judge(prompt, system_prompt=GRADER_SYSTEM_PROMPT)
+
+        # parse grading response
+        match = re.search(r'(m|M)', judge_response)
+        res = match.group(0) if match else None
+
+        if res:
+            winrate = 1 if res == 'M' else 0
+        else:
+            logger.info(f'Failed to parse grading response: {prompt=}\n {judge_response=}')
+            winrate = 0
+
+        # Set score based on the match result
+        score.value = {'winrate': winrate}
+        score.explanation = f'LLM judge: {judge_response}'
+        score.metadata = {
+            'source': 'llm_judge',
+            'judge_strategy': self.judge_strategy,
+            'model': self.llm_judge.model_id
+        }
+        score.main_score_name = 'winrate'
+        return score

evalscope/benchmarks/amc/amc_adapter.py

@@ -0,0 +1,51 @@
+from typing import Any, Dict
+
+from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='amc',
+        pretty_name='AMC',
+        tags=[Tags.MATH, Tags.REASONING],
+        description=
+        'AMC (American Mathematics Competitions) is a series of mathematics competitions for high school students.',
+        dataset_id='evalscope/amc_22-24',
+        subset_list=['amc22', 'amc23', 'amc24'],
+        metric_list=[{
+            'acc': {
+                'numeric': True
+            }
+        }],
+        prompt_template='{question}\nPlease reason step by step, and put your final answer within \\boxed{{}}.',
+    )
+)
+class AMCAdapter(DefaultDataAdapter):
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        # Use split as subset
+        self.split_as_subset = True
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        return Sample(
+            input=record['problem'],
+            target=record['answer'],
+            metadata={
+                'year': record['year'],
+                'url': record['url'],
+                'solution': record.get('solution', '')
+            },
+        )
+
+    def extract_answer(self, prediction: str, task_state):
+        from evalscope.metrics.math_parser import extract_answer
+
+        return extract_answer(prediction)

evalscope/benchmarks/arc/arc_adapter.py

@@ -1,161 +1,46 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 
-import json
-import os
-
-from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.constants import EvalType
-from evalscope.metrics import AverageAccuracy, exact_match
-from evalscope.models import MultiChoiceModelAdapter
-from evalscope.utils import ResponseParser
+from evalscope.api.benchmark import BenchmarkMeta, MultiChoiceAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
 from evalscope.utils.logger import get_logger
-
-# flake8: noqa
+from evalscope.utils.multi_choices import MultipleChoiceTemplate
 
 logger = get_logger()
 
 
-@Benchmark.register(
-    name='arc',
-    dataset_id='modelscope/ai2_arc',
-    model_adapter=MultiChoiceModelAdapter,
-    subset_list=['ARC-Easy', 'ARC-Challenge'],
-    metric_list=[AverageAccuracy],
-    few_shot_num=0,
-    train_split='train',
-    eval_split='test',
-    prompt_template='',
+@register_benchmark(
+    BenchmarkMeta(
+        name='arc',
+        pretty_name='ARC',
+        tags=[Tags.REASONING, Tags.MULTIPLE_CHOICE],
+        description=
+        'The ARC (AI2 Reasoning Challenge) benchmark is designed to evaluate the reasoning capabilities of AI models through multiple-choice questions derived from science exams. It includes two subsets: ARC-Easy and ARC-Challenge, which vary in difficulty.',  # noqa: E501
+        dataset_id='allenai/ai2_arc',
+        subset_list=['ARC-Easy', 'ARC-Challenge'],
+        metric_list=['acc'],
+        few_shot_num=0,
+        train_split='train',
+        eval_split='test',
+        prompt_template=MultipleChoiceTemplate.SINGLE_ANSWER,
+    )
 )
-class ARCAdapter(DataAdapter):
-
-    choices = ['A', 'B', 'C', 'D']
+class ARCAdapter(MultiChoiceAdapter):
 
     def __init__(self, **kwargs):
-        few_shot_num = kwargs.get('few_shot_num', None)
-        if few_shot_num is None:
-            # Use 0-shot by default
-            logger.info(f'Set 0-shot examples by system for ARC.')
-            few_shot_num = 0
-
-        if few_shot_num != 0:
-            logger.warning(f'few_shot_num is recommended to set 0 for ARC, got {few_shot_num}.')
-
         super().__init__(**kwargs)
 
-    def load_from_disk(self, dataset_name_or_path, subset_list, work_dir, **kwargs) -> dict:
-        """
-        Load the dataset from local disk.
-
-        dataset_name_or_path: str, the dataset id or path. e.g. 'arc'
-        subset_list: list, the subset list to load. e.g. ['ARC-Easy', 'ARC-Challenge']
-        work_dir: str, the local root data directory. e.g. '/path/to/data'
-        kwargs: dict, other arguments.
-        """
-        data_dict = {}
-        for subset_name in subset_list:
-            if os.path.exists(dataset_name_or_path):
-                subset_path = os.path.join(dataset_name_or_path, subset_name)
-            else:
-                subset_path = os.path.join(work_dir, dataset_name_or_path, subset_name)
-            for split_name in ['Train', 'Test']:
-                split_path = os.path.join(subset_path, f'{subset_name}-{split_name}.jsonl')
-                if os.path.exists(split_path):
-                    with open(split_path, 'r', errors='ignore') as in_f:
-                        rows = []
-                        for line in in_f:
-                            item = json.loads(line.strip())
-                            raw_choices = item['question']['choices']
-                            rows.append({
-                                'id': item['id'],
-                                'question': item['question']['stem'],
-                                'choices': {
-                                    'text': [d['text'] for d in raw_choices],
-                                    'label': [d['label'] for d in raw_choices]
-                                },
-                                'answerKey': item['answerKey'],
-                            })
-
-                        if subset_name in data_dict:
-                            data_dict[subset_name].update({split_name.lower(): rows})
-                        else:
-                            data_dict[subset_name] = {split_name.lower(): rows}
-
-        return data_dict
-
-    def gen_prompt(self, input_d: dict, few_shot_list: list, **kwargs) -> dict:
-        """
-        Generate model prompt from raw data, unify the prompt format for ARC benchmark.
-
-        Args:
-            input_d (dict): The raw input. A single data format of the ARC:
-
-            {
-                'id': 'Mercury_7220990',
-                'question': 'Which factor will most likely cause a person to develop a fever?',
-                'choices':
-                    {
-                        'text':['a leg muscle relaxing after exercise',
-                                'a bacterial population in the bloodstream',
-                                'several viral particles on the skin',
-                                'carbohydrates being digested in the stomach'],
-                        'label': ['A', 'B', 'C', 'D']
-                    },
-                'answerKey': 'B'
-            }
-
-        Returns:
-            {'data': ['xxx'], 'multi_choices': ['A', 'B', 'C', 'D']}
-        """
-        few_shot_prompts = [self._generate_prompt(input_d=sample, include_answer=True) for sample in few_shot_list]
-        context: str = '\n'.join(few_shot_prompts)
-
-        # context = f'The following are multiple choice questions, please output correct answer in the form of A or B or C or D, do not output explanation:\n {context}'
-        full_prompt: str = context + self._generate_prompt(input_d=input_d, include_answer=False)
-
-        return {'data': [full_prompt], 'multi_choices': self.choices, 'system_prompt': self.prompt_template}
-
-    def get_gold_answer(self, input_d: dict) -> str:
-        # Get the gold choice
-        return input_d.get('answerKey', '')
-
-    def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = EvalType.CHECKPOINT) -> str:
-        """
-        Parse the model output to get the answer. Could be the best choice index.
-
-        Args:
-            result: Predicted answer from the model. Usually a string for chat.
-            raw_input_d (dict): The raw input. Depending on the dataset.
-            eval_type: 'checkpoint' or 'service' or `custom`, default: 'checkpoint'
-
-        Returns:
-            The parsed answer. Depending on the dataset. Usually a string for chat.
-        """
-        if eval_type == EvalType.CHECKPOINT:
-            return result
-        elif eval_type == EvalType.SERVICE:
-            return ResponseParser.parse_first_option_with_choices(
-                text=result, options=self.choices)  # TODO: to be checked !
-        elif eval_type == EvalType.CUSTOM:
-            return ResponseParser.parse_first_option_with_choices(
-                text=result, options=self.choices)  # TODO: to be checked !
-        else:
-            raise ValueError(f'Invalid eval_type: {eval_type}')
-
-    def match(self, gold: str, pred: str) -> float:
-        return exact_match(gold=gold, pred=pred)
-
-    @classmethod
-    def _generate_prompt(cls, input_d: dict, include_answer=True) -> str:
-
-        example: str = input_d['question']
-
-        choices_texts: list = input_d['choices']['text']
-        choices_labels: list = input_d['choices']['label']
-        choices_prompts: str = '\n'.join([label + '. ' + text for text, label in zip(choices_texts, choices_labels)])
-        example += '\n' + choices_prompts
-
-        example += '\nAnswer:'
-        if include_answer:
-            example += ' {}\n\n'.format(input_d['answerKey'])
-
-        return example
+    def record_to_sample(self, record) -> Sample:
+        # Convert choice labels to indices (A->0, B->1, etc.)
+        choice_texts = record['choices']['text']
+        answer_key = record['answerKey']
+
+        return Sample(
+            input=record['question'],
+            choices=choice_texts,
+            target=answer_key,
+            metadata={
+                'id': record.get('id', ''),
+            },
+        )