evalscope 0.17.1__py3-none-any.whl → 1.0.1__py3-none-any.whl

evalscope/benchmarks/arc/arc_adapter.py

@@ -1,159 +1,46 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 
-import json
-import os
-
-from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.constants import EvalType, OutputType
-from evalscope.metrics import exact_match
-from evalscope.metrics.completion_parsers import ResponseParser
+from evalscope.api.benchmark import BenchmarkMeta, MultiChoiceAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
 from evalscope.utils.logger import get_logger
-
-# flake8: noqa
+from evalscope.utils.multi_choices import MultipleChoiceTemplate
 
 logger = get_logger()
 
 
-@Benchmark.register(
-    name='arc',
-    pretty_name='ARC',
-    tags=['Reasoning', 'MCQ'],
-    description=
-    'The ARC (AI2 Reasoning Challenge) benchmark is designed to evaluate the reasoning capabilities of AI models through multiple-choice questions derived from science exams. It includes two subsets: ARC-Easy and ARC-Challenge, which vary in difficulty.',  # noqa: E501
-    dataset_id='modelscope/ai2_arc',
-    model_adapter=OutputType.GENERATION,
-    output_types=[OutputType.MULTIPLE_CHOICE, OutputType.GENERATION],
-    subset_list=['ARC-Easy', 'ARC-Challenge'],
-    metric_list=['AverageAccuracy'],
-    few_shot_num=0,
-    train_split='train',
-    eval_split='test',
-    prompt_template=
-    'Given the following question and four candidate answers (A, B, C and D), choose the best answer.\n{query}\nYour response should end with "The best answer is [the_answer_letter]" where the [the_answer_letter] is one of A, B, C or D.',  # noqa
+@register_benchmark(
+    BenchmarkMeta(
+        name='arc',
+        pretty_name='ARC',
+        tags=[Tags.REASONING, Tags.MULTIPLE_CHOICE],
+        description=
+        'The ARC (AI2 Reasoning Challenge) benchmark is designed to evaluate the reasoning capabilities of AI models through multiple-choice questions derived from science exams. It includes two subsets: ARC-Easy and ARC-Challenge, which vary in difficulty.',  # noqa: E501
+        dataset_id='allenai/ai2_arc',
+        subset_list=['ARC-Easy', 'ARC-Challenge'],
+        metric_list=['acc'],
+        few_shot_num=0,
+        train_split='train',
+        eval_split='test',
+        prompt_template=MultipleChoiceTemplate.SINGLE_ANSWER,
+    )
 )
-class ARCAdapter(DataAdapter):
+class ARCAdapter(MultiChoiceAdapter):
 
     def __init__(self, **kwargs):
-        few_shot_num = kwargs.get('few_shot_num', None)
-        if few_shot_num is None:
-            # Use 0-shot by default
-            logger.info(f'Set 0-shot examples by system for ARC.')
-            few_shot_num = 0
-
-        if few_shot_num != 0:
-            logger.warning(f'few_shot_num is recommended to set 0 for ARC, got {few_shot_num}.')
-
         super().__init__(**kwargs)
 
-        self.choices = ['A', 'B', 'C', 'D']
-
-    def load_from_disk(self, dataset_name_or_path, subset_list, work_dir, **kwargs) -> dict:
-        """
-        Load the dataset from local disk.
-
-        dataset_name_or_path: str, the dataset id or path. e.g. 'arc'
-        subset_list: list, the subset list to load. e.g. ['ARC-Easy', 'ARC-Challenge']
-        work_dir: str, the local root data directory. e.g. '/path/to/data'
-        kwargs: dict, other arguments.
-        """
-        data_dict = {}
-        for subset_name in subset_list:
-            if os.path.exists(dataset_name_or_path):
-                subset_path = os.path.join(dataset_name_or_path, subset_name)
-            else:
-                subset_path = os.path.join(work_dir, dataset_name_or_path, subset_name)
-            for split_name in ['Train', 'Test']:
-                split_path = os.path.join(subset_path, f'{subset_name}-{split_name}.jsonl')
-                if os.path.exists(split_path):
-                    with open(split_path, 'r', errors='ignore', encoding='utf-8') as in_f:
-                        rows = []
-                        for line in in_f:
-                            item = json.loads(line.strip())
-                            raw_choices = item['question']['choices']
-                            rows.append({
-                                'id': item['id'],
-                                'question': item['question']['stem'],
-                                'choices': {
-                                    'text': [d['text'] for d in raw_choices],
-                                    'label': [d['label'] for d in raw_choices]
-                                },
-                                'answerKey': item['answerKey'],
-                            })
-
-                        if subset_name in data_dict:
-                            data_dict[subset_name].update({split_name.lower(): rows})
-                        else:
-                            data_dict[subset_name] = {split_name.lower(): rows}
-
-        return data_dict
-
-    def gen_prompt(self, input_d: dict, few_shot_list: list, **kwargs) -> dict:
-        """
-        Generate model prompt from raw data, unify the prompt format for ARC benchmark.
-
-        Args:
-            input_d (dict): The raw input. A single data format of the ARC:
-
-            {
-                'id': 'Mercury_7220990',
-                'question': 'Which factor will most likely cause a person to develop a fever?',
-                'choices':
-                    {
-                        'text':['a leg muscle relaxing after exercise',
-                                'a bacterial population in the bloodstream',
-                                'several viral particles on the skin',
-                                'carbohydrates being digested in the stomach'],
-                        'label': ['A', 'B', 'C', 'D']
-                    },
-                'answerKey': 'B'
-            }
-
-        Returns:
-            {'data': ['xxx'], 'multi_choices': ['A', 'B', 'C', 'D']}
-        """
-        few_shot_prompts = [self._generate_prompt(input_d=sample, include_answer=True) for sample in few_shot_list]
-        context = '\n'.join(few_shot_prompts) + self._generate_prompt(input_d=input_d, include_answer=False)
-
-        full_prompt = self.prompt_template.format(query=context)
-
-        return self.gen_prompt_data(full_prompt)
-
-    def get_gold_answer(self, input_d: dict) -> str:
-        # Get the gold choice
-        return input_d.get('answerKey', '')
-
-    def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = EvalType.CHECKPOINT) -> str:
-        """
-        Parse the model output to get the answer. Could be the best choice index.
-
-        Args:
-            result: Predicted answer from the model. Usually a string for chat.
-            raw_input_d (dict): The raw input. Depending on the dataset.
-            eval_type: 'checkpoint' or 'service' or `custom`, default: 'checkpoint'
-
-        Returns:
-            The parsed answer. Depending on the dataset. Usually a string for chat.
-        """
-        if self.model_adapter == OutputType.MULTIPLE_CHOICE:
-            return result
-        else:
-            return ResponseParser.parse_first_option(text=result, options=self.choices)
-
-    def match(self, gold: str, pred: str) -> float:
-        return exact_match(gold=gold, pred=pred)
-
-    @classmethod
-    def _generate_prompt(cls, input_d: dict, include_answer=True) -> str:
-
-        example: str = input_d['question']
-
-        choices_texts: list = input_d['choices']['text']
-        choices_labels: list = input_d['choices']['label']
-        choices_prompts: str = '\n'.join([label + '. ' + text for text, label in zip(choices_texts, choices_labels)])
-        example += '\n' + choices_prompts
-
-        if include_answer:
-            example += '\nAnswer:'
-            example += ' {}\n\n'.format(input_d['answerKey'])
-
-        return example
+    def record_to_sample(self, record) -> Sample:
+        # Convert choice labels to indices (A->0, B->1, etc.)
+        choice_texts = record['choices']['text']
+        answer_key = record['answerKey']
+
+        return Sample(
+            input=record['question'],
+            choices=choice_texts,
+            target=answer_key,
+            metadata={
+                'id': record.get('id', ''),
+            },
+        )

evalscope/benchmarks/arena_hard/arena_hard_adapter.py

@@ -1,75 +1,97 @@
-from typing import Any, List
-
-from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.metrics import LLMJudge, Metric, mean, metric_registry
+# flake8: noqa: E501
+from typing import Any, Dict, List
+
+from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.evaluator import TaskState
+from evalscope.api.metric import AggScore, SampleScore, Score
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
 from evalscope.utils.logger import get_logger
 
-# flake8: noqa
-
 logger = get_logger()
 
-GRADER_SYSTEM_PROMPT = "Please act as an impartial judge and evaluate the quality of the responses provided by two AI assistants to the user prompt displayed below. You will be given assistant A's answer and assistant B's answer. Your job is to evaluate which assistant's answer is better.\n\nBegin your evaluation by generating your own answer to the prompt. You must provide your answers before judging any answers.\n\nWhen evaluating the assistants' answers, compare both assistants' answers with your answer. You must identify and correct any mistakes or inaccurate information.\n\nThen consider if the assistant's answers are helpful, relevant, and concise. Helpful means the answer correctly responds to the prompt or follows the instructions. Note when user prompt has any ambiguity or more than one interpretation, it is more helpful and appropriate to ask for clarifications or more information from the user than providing an answer based on assumptions. Relevant means all parts of the response closely connect or are appropriate to what is being asked. Concise means the response is clear and not verbose or excessive.\n\nThen consider the creativity and novelty of the assistant's answers when needed. Finally, identify any missing important information in the assistants' answers that would be beneficial to include when responding to the user prompt.\n\nAfter providing your explanation, you must output only one of the following choices as your final verdict with a label:\n\n1. Assistant A is significantly better: [[A>>B]]\n2. Assistant A is slightly better: [[A>B]]\n3. Tie, relatively the same: [[A=B]]\n4. Assistant B is slightly better: [[B>A]]\n5. Assistant B is significantly better: [[B>>A]]\n\nExample output: \"My final verdict is tie: [[A=B]]\"."  # noqa: E501
-
-GRADER_TEMPLATE = "<|User Prompt|>\n{question}\n\n<|The Start of Assistant A's Answer|>\n{answer_1}\n<|The End of Assistant A's Answer|>\n\n<|The Start of Assistant B's Answer|>\n{answer_2}\n<|The End of Assistant B's Answer|>".strip(
-)  # noqa: E501
-
-
-@Benchmark.register(
-    name='arena_hard',
-    pretty_name='ArenaHard',
-    tags=['Instruction-Following', 'Arena'],
-    description=
-    'ArenaHard is a benchmark designed to evaluate the performance of large language models in a competitive setting, '
-    'where models are pitted against each other in a series of tasks to determine their relative strengths and weaknesses. '
-    'It includes a set of challenging tasks that require reasoning, understanding, and generation capabilities. '
-    'Currently not support `style-controlled winrate`; the official Judge model is `gpt-4-1106-preview`, while the baseline model is `gpt-4-0314`.',  # noqa: E501
-    dataset_id='AI-ModelScope/arena-hard-auto-v0.1',
-    metric_list=['winrate'],
-    few_shot_num=0,
-    train_split=None,
-    eval_split='test')
-class ArenaHardAdapter(DataAdapter):
+GRADER_SYSTEM_PROMPT = """Please act as an impartial judge and evaluate the quality of the responses provided by two AI assistants to the user prompt displayed below. You will be given assistant A's answer and assistant B's answer. Your job is to evaluate which assistant's answer is better.\n\nBegin your evaluation by generating your own answer to the prompt. You must provide your answers before judging any answers.\n\nWhen evaluating the assistants' answers, compare both assistants' answers with your answer. You must identify and correct any mistakes or inaccurate information.\n\nThen consider if the assistant's answers are helpful, relevant, and concise. Helpful means the answer correctly responds to the prompt or follows the instructions. Note when user prompt has any ambiguity or more than one interpretation, it is more helpful and appropriate to ask for clarifications or more information from the user than providing an answer based on assumptions. Relevant means all parts of the response closely connect or are appropriate to what is being asked. Concise means the response is clear and not verbose or excessive.\n\nThen consider the creativity and novelty of the assistant's answers when needed. Finally, identify any missing important information in the assistants' answers that would be beneficial to include when responding to the user prompt.\n\nAfter providing your explanation, you must output only one of the following choices as your final verdict with a label:\n\n1. Assistant A is significantly better: [[A>>B]]\n2. Assistant A is slightly better: [[A>B]]\n3. Tie, relatively the same: [[A=B]]\n4. Assistant B is slightly better: [[B>A]]\n5. Assistant B is significantly better: [[B>>A]]\n\nExample output: \"My final verdict is tie: [[A=B]]\"."""  # noqa: E501
+
+GRADER_TEMPLATE = """<|User Prompt|>\n{question}\n\n<|The Start of Assistant A's Answer|>\n{answer_1}\n<|The End of Assistant A's Answer|>\n\n<|The Start of Assistant B's Answer|>\n{answer_2}\n<|The End of Assistant B's Answer|>""".strip(
+)
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='arena_hard',
+        pretty_name='ArenaHard',
+        tags=[Tags.INSTRUCTION_FOLLOWING, Tags.ARENA],
+        description=
+        'ArenaHard is a benchmark designed to evaluate the performance of large language models in a competitive setting, '
+        'where models are pitted against each other in a series of tasks to determine their relative strengths and weaknesses. '
+        'It includes a set of challenging tasks that require reasoning, understanding, and generation capabilities. '
+        'Currently not support `style-controlled winrate`; the official Judge model is `gpt-4-1106-preview`, while the baseline model is `gpt-4-0314`.',
+        dataset_id='AI-ModelScope/arena-hard-auto-v0.1',
+        metric_list=['winrate'],
+        few_shot_num=0,
+        train_split=None,
+        eval_split='test',
+        prompt_template='{question}'
+    )
+)
+class ArenaHardAdapter(DefaultDataAdapter):
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
 
-        # register metrics
-        metric_registry.register(Metric(name='winrate', object=mean))
-
-        # whether to use LLM as a judge
-        self.llm_as_a_judge = True
+        self._use_llm_judge = True  # Use LLM as a judge by default
 
-    def gen_prompt(self, input_d: dict, subset_name: str, few_shot_list: list, **kwargs) -> dict:
-        question = input_d['question']
-        return self.gen_prompt_data(question)
-
-    def get_gold_answer(self, input_d: dict) -> str:
-        return input_d['prediction']
-
-    def parse_pred_result(self, result: str, raw_input_d: dict = None, **kwargs) -> str:
-        return result.strip()
-
-    def match(self, gold: str, pred: str):
-        # simple match
-        logger.warning(f'Please use LLMJudge to match the result for {self.name}')
-        return None
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        """
+        Convert a data record to a Sample object.
 
-    def llm_match(self, gold: Any, pred: Any, judge: LLMJudge, **kwargs) -> dict:
-        from .utils import post_process_arenahard
+        Args:
+            record (Dict[str, Any]): Input data record.
 
-        raw_input = kwargs.get('raw_input', None)
-        question = raw_input['question']
-        # gold is baseline answer 'A', pred is model answer 'B'
-        prompt1 = GRADER_TEMPLATE.format(question=question, answer_1=gold, answer_2=pred)
+        Returns:
+            Sample: Sample object with input, target, and metadata.
+        """
+        question = record['question']
+        baseline_prediction = record['prediction']  # baseline model prediction
+
+        return Sample(
+            input=question, target=baseline_prediction, metadata={'capability': record.get('capability', 'unknown')}
+        )
+
+    def llm_match_score(
+        self,
+        original_prediction: str,
+        filtered_prediction: str,
+        reference: str,
+        task_state: TaskState,
+    ) -> Score:
+        from .utils import get_judge_score, post_process_arenahard
+
+        score = Score(
+            extracted_prediction=filtered_prediction,
+            prediction=original_prediction,
+        )
+
+        question = task_state.input_text
+
+        # reference is baseline answer 'A', filtered_prediction is model answer 'B'
+        prompt1 = GRADER_TEMPLATE.format(question=question, answer_1=reference, answer_2=filtered_prediction)
         # reverse the order
-        prompt2 = GRADER_TEMPLATE.format(question=question, answer_1=pred, answer_2=gold)
+        prompt2 = GRADER_TEMPLATE.format(question=question, answer_1=filtered_prediction, answer_2=reference)
+
         # get grading response
-        game1_response = judge(prompt1, system_prompt=GRADER_SYSTEM_PROMPT)
-        game2_response = judge(prompt2, system_prompt=GRADER_SYSTEM_PROMPT)
+        game1_response = self.llm_judge.judge(prompt1, system_prompt=GRADER_SYSTEM_PROMPT)
+        game2_response = self.llm_judge.judge(prompt2, system_prompt=GRADER_SYSTEM_PROMPT)
+
         # parse grading response
         res1 = post_process_arenahard(game1_response)
         res2 = post_process_arenahard(game2_response)
-        return {
+
+        score1 = get_judge_score(res1, reverse=True)
+        score2 = get_judge_score(res2, reverse=False)
+
+        battle_result = {
             'model_a':
             'gpt4-0314',
             'model_b':
@@ -88,22 +110,26 @@ class ArenaHardAdapter(DataAdapter):
             ]
         }
 
-    def compute_metric(self, review_res_list: List[dict], **kwargs) -> List[dict]:
-        """
-        compute score of the model
-        """
+        # Set score based on the battle result
+        score.value = {'score': (score1 + score2) / 2}
+        score.explanation = f'LLM judge battles: Game1: {game1_response[:100]}... Game2: {game2_response[:100]}...'
+        score.metadata = {
+            'source': 'llm_judge',
+            'judge_strategy': self.judge_strategy,
+            'model': self.llm_judge.model_id,
+            'battle_result': battle_result
+        }
+        return score
+
+    def aggregate_scores(self, sample_scores: List[SampleScore]) -> List[AggScore]:
         import pandas as pd
 
         from .utils import compute_mle_elo, get_battles_from_row, get_bootstrap_result, get_win_rate_column
 
-        if isinstance(review_res_list[0], list):
-            review_res_list = [item for sublist in review_res_list for item in sublist]
-
-        battles = pd.concat([get_battles_from_row(res) for res in review_res_list])
+        battles = pd.concat([get_battles_from_row(res.score.metadata['battle_result']) for res in sample_scores])
 
         bootstrap_online_elo = compute_mle_elo(battles)
 
-        # bootstrap_elo_lu = get_bootstrap_result(battles, compute_mle_elo, 100)
         stats = pd.DataFrame()
         stats['results'] = None
         stats['results'] = stats['results'].astype('object')
@@ -112,11 +138,11 @@ class ArenaHardAdapter(DataAdapter):
             # assert model in bootstrap_elo_lu.columns
             stats.at[i, 'model'] = model
             stats.at[i, 'score'] = bootstrap_online_elo[model]
-            # stats.at[i, "lower"] = np.percentile(bootstrap_elo_lu[model], 2.5)
-            # stats.at[i, "upper"] = np.percentile(bootstrap_elo_lu[model], 97.5)
-
-        # stats['score'] = get_win_rate_column(stats, 'score', 'gpt4-0314').tolist()
 
         score = get_win_rate_column(stats, 'score', 'gpt4-0314').at['test_model']
 
-        return [{'metric_name': 'winrate', 'score': score, 'num': len(review_res_list)}]
+        return [AggScore(
+            score=score,
+            metric_name='winrate',
+            num=len(sample_scores),
+        )]

evalscope/benchmarks/arena_hard/utils.py

@@ -19,6 +19,41 @@ def post_process_arenahard(completion):
         return None
 
 
+def get_judge_score(result, reverse=False):
+    """
+    Calculate the judge score, considering confidence weight.
+
+    Args:
+        result: Judgment result ('A=B', 'A>B', 'A>>B', 'B>A', 'B>>A')
+        reverse: Whether to reverse the score
+
+    Returns:
+        float: Weighted score
+    """
+
+    # Base score mapping - using finer-grained scores
+    if not reverse:
+        score_mapping = {
+            'A=B': 0.5,  # Tie
+            'A>B': 0.75,  # A slightly wins
+            'A>>B': 1.0,  # A significantly wins
+            'B>A': 0.25,  # B slightly wins
+            'B>>A': 0.0,  # B significantly wins
+        }
+    else:
+        score_mapping = {
+            'A=B': 0.5,  # Tie
+            'A>B': 0.25,  # A slightly wins
+            'A>>B': 0.0,  # A significantly wins
+            'B>A': 0.75,  # B slightly wins
+            'B>>A': 1.0,  # B significantly wins
+        }
+
+    base_score = score_mapping.get(result, 0.5)
+
+    return base_score
+
+
 def get_battles_from_row(row, first_game_only=False, multiplier=3):
     results = []
     output = {'model_a': row['model_a'], 'model_b': row['model_b']}
@@ -106,7 +141,8 @@ def compute_mle_elo(df, SCALE=400, BASE=10, INIT_RATING=1000):
         return elo_scores.sort_values(ascending=False)
 
     lr = LogisticRegression(
-        fit_intercept=False, penalty=None, tol=1e-8)  # May need to set a small value when not use GPT4 as judge model
+        fit_intercept=False, penalty=None, tol=1e-8
+    )  # May need to set a small value when not use GPT4 as judge model
     lr.fit(X, Y)
 
     elo_scores = SCALE * lr.coef_[0] + INIT_RATING