evalscope 0.8.2__py3-none-any.whl → 0.10.0__py3-none-any.whl

evalscope/benchmarks/ifeval/utils.py

@@ -0,0 +1,134 @@
+import dataclasses
+from typing import Dict, Optional, Union
+
+from evalscope.benchmarks.ifeval import instructions_registry
+
+
+@dataclasses.dataclass
+class InputExample:
+    key: int
+    instruction_id_list: list[str]
+    prompt: str
+    kwargs: list[Dict[str, Optional[Union[str, int]]]]
+
+
+@dataclasses.dataclass
+class OutputExample:
+    instruction_id_list: list[str]
+    prompt: str
+    response: str
+    follow_all_instructions: bool
+    follow_instruction_list: list[bool]
+
+
+def test_instruction_following_strict(
+    inp,
+    response,
+):
+    """Tests response to see if instructions are followed."""
+    instruction_list = inp.instruction_id_list
+    is_following_list = []
+
+    for index, instruction_id in enumerate(instruction_list):
+        instruction_cls = instructions_registry.INSTRUCTION_DICT[instruction_id]
+        instruction = instruction_cls(instruction_id)
+
+        # Remove None values from kwargs to avoid unexpected keyword argument errors in build_description method.
+        kwargs = {k: v for k, v in inp.kwargs[index].items() if v}
+        instruction.build_description(**kwargs)
+        args = instruction.get_instruction_args()
+        if args and 'prompt' in args:
+            instruction.build_description(prompt=inp.prompt)
+
+        if response.strip() and instruction.check_following(response):
+            is_following_list.append(True)
+        else:
+            is_following_list.append(False)
+
+    return OutputExample(
+        instruction_id_list=inp.instruction_id_list,
+        prompt=inp.prompt,
+        response=response,
+        follow_all_instructions=all(is_following_list),
+        follow_instruction_list=is_following_list,
+    )
+
+
+def test_instruction_following_loose(
+    inp,
+    response,
+):
+    """Tests response for an upper bound for following instructions."""
+    r = response.split('\n')
+    response_remove_first = '\n'.join(r[1:]).strip()
+    response_remove_last = '\n'.join(r[:-1]).strip()
+    response_remove_both = '\n'.join(r[1:-1]).strip()
+    revised_response = response.replace('*', '')
+    revised_response_remove_first = response_remove_first.replace('*', '')
+    revised_response_remove_last = response_remove_last.replace('*', '')
+    revised_response_remove_both = response_remove_both.replace('*', '')
+    all_responses = [
+        response,
+        revised_response,
+        response_remove_first,
+        response_remove_last,
+        response_remove_both,
+        revised_response_remove_first,
+        revised_response_remove_last,
+        revised_response_remove_both,
+    ]
+    instruction_list = inp.instruction_id_list
+    is_following_list = []
+
+    for index, instruction_id in enumerate(instruction_list):
+        instruction_cls = instructions_registry.INSTRUCTION_DICT[instruction_id]
+        instruction = instruction_cls(instruction_id)
+
+        # Remove None values from kwargs to avoid unexpected keyword argument errors in build_description method.
+        kwargs = {k: v for k, v in inp.kwargs[index].items() if v}
+        instruction.build_description(**kwargs)
+        args = instruction.get_instruction_args()
+        if args and 'prompt' in args:
+            instruction.build_description(prompt=inp.prompt)
+
+        is_following = False
+        for r in all_responses:
+            if r.strip() and instruction.check_following(r):
+                is_following = True
+                break
+
+        is_following_list.append(is_following)
+
+    return OutputExample(
+        instruction_id_list=inp.instruction_id_list,
+        prompt=inp.prompt,
+        response=response,
+        follow_all_instructions=all(is_following_list),
+        follow_instruction_list=is_following_list,
+    )
+
+
+def process_results(doc, results):
+    inp = InputExample(
+        key=doc['key'],
+        instruction_id_list=doc['instruction_id_list'],
+        prompt=doc['prompt'],
+        kwargs=doc['kwargs'],
+    )
+    response = results[0]
+
+    out_strict = test_instruction_following_strict(inp, response)
+    out_loose = test_instruction_following_loose(inp, response)
+
+    return {
+        'prompt_level_strict_acc': out_strict.follow_all_instructions,
+        'inst_level_strict_acc': out_strict.follow_instruction_list,
+        'prompt_level_loose_acc': out_loose.follow_all_instructions,
+        'inst_level_loose_acc': out_loose.follow_instruction_list,
+    }
+
+
+def agg_inst_level_acc(items):
+    flat_items = [item for sublist in items for item in sublist]
+    inst_level_acc = sum(flat_items) / len(flat_items)
+    return inst_level_acc

evalscope/benchmarks/iquiz/iquiz_adapter.py

@@ -0,0 +1,63 @@
+from evalscope.benchmarks import Benchmark, DataAdapter
+from evalscope.constants import AnswerKeys, EvalType
+from evalscope.metrics import AverageAccuracy, exact_match
+from evalscope.models import ChatGenerationModelAdapter
+from evalscope.utils.utils import ResponseParser
+
+
+@Benchmark.register(
+    name='iquiz',
+    dataset_id='AI-ModelScope/IQuiz',
+    model_adapter=ChatGenerationModelAdapter,
+    subset_list=['IQ', 'EQ'],
+    metric_list=[AverageAccuracy],
+    few_shot_num=0,
+    train_split=None,
+    eval_split='test',
+    prompt_template='你是一个高智商和高情商的专家，你被要求回答一个选择题，并选出一个正确的选项，解释原因，最终输出格式为：`答案是(选项)`。',  # noqa: E501
+)
+class IQuizAdapter(DataAdapter):
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+        self.choices = ['A', 'B', 'C', 'D', 'E']
+
+    def gen_prompt(self, input_d: dict, subset_name: str, few_shot_list: list, **kwargs) -> dict:
+        """
+        Generate model prompt from input data.
+        example:
+        {
+            "question":"天气预报说本周星期三会下雨，昨天果然下雨了，今天星期几？",
+            "choices":["星期一","星期二","星期三","星期四"],
+            "answer":"D",
+            "level":1
+        }
+        """
+        prompt = f"问题: {input_d['question']}\n"
+        prompt += self.__form_options(input_d['choices'])
+        return {'data': [prompt], 'multi_choices': self.choices, 'system_prompt': self.prompt_template}
+
+    def __form_options(self, options: list):
+        option_str = '选项:\n'
+        for opt, choice in zip(options, self.choices):
+            option_str += f'({choice}): {opt}' + '\n'
+        return option_str
+
+    def get_gold_answer(self, input_d: dict) -> str:
+        """
+        Parse the raw input labels (gold).
+        """
+        return input_d['answer']
+
+    def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = EvalType.CHECKPOINT) -> str:
+        """
+        Parse the predicted result and extract proper answer.
+        """
+        return ResponseParser.parse_first_option_with_choices(result, self.choices)
+
+    def match(self, gold: str, pred: str) -> float:
+        """
+        Match the gold answer and the predicted answer.
+        """
+        return exact_match(gold=gold, pred=pred)

evalscope/benchmarks/mmlu/__init__.py

@@ -1,6 +1 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
-
-from evalscope.benchmarks.mmlu.mmlu_adapter import DATASET_ID, SUBJECT_MAPPING, SUBSET_LIST
-from evalscope.benchmarks.mmlu.mmlu_adapter import MMLUAdapter
-from evalscope.benchmarks.mmlu.mmlu_adapter import MMLUAdapter as DataAdapterClass
-from evalscope.models.model_adapter import MultiChoiceModelAdapter as ModelAdapterClass  # noqa

evalscope/benchmarks/mmlu/mmlu_adapter.py

@@ -2,8 +2,10 @@
 import csv
 import os
 
-from evalscope.benchmarks.data_adapter import DataAdapter
-from evalscope.metrics.metrics import exact_match, weighted_mean
+from evalscope.benchmarks import Benchmark, DataAdapter
+from evalscope.constants import EvalType
+from evalscope.metrics import AverageAccuracy, exact_match
+from evalscope.models import MultiChoiceModelAdapter
 from evalscope.utils import ResponseParser, normalize_score
 from evalscope.utils.logger import get_logger
 
@@ -134,40 +136,31 @@ SUBJECT_MAPPING = {
 }
 
 
+@Benchmark.register(
+    name='mmlu',
+    dataset_id='modelscope/mmlu',
+    model_adapter=MultiChoiceModelAdapter,
+    subset_list=SUBSET_LIST,
+    metric_list=[AverageAccuracy],
+    few_shot_num=5,
+    train_split='train',
+    eval_split='test',
+    prompt_template='',
+)
 class MMLUAdapter(DataAdapter):
 
     choices = ['A', 'B', 'C', 'D']
 
-    def __init__(self,
-                 subset_list: list = None,
-                 metric_list: list = None,
-                 few_shot_num: int = None,
-                 train_split: str = 'train',
-                 eval_split: str = 'test',
-                 **kwargs):
-
-        if subset_list is None:
-            subset_list = SUBSET_LIST
-
-        if metric_list is None:
-            metric_list = [{'name': 'WeightedAverageAccuracy', 'object': weighted_mean}]
-
-        if few_shot_num is None:
-            # Use 5-shot by default
-            logger.info(f'Set 5-shot examples by system for MMLU.')
-            few_shot_num = 5
+    def __init__(self, **kwargs):
 
+        few_shot_num = kwargs.get('few_shot_num', 5)
         if few_shot_num > 5:
             logger.warning(f'few_shot_num <= 5 for MMLU, but got {few_shot_num}. Use 5-shot by default.')
-            few_shot_num = 5
+            kwargs['few_shot_num'] = 5
 
-        super().__init__(
-            subset_list=subset_list,
-            metric_list=metric_list,
-            few_shot_num=few_shot_num,
-            train_split=train_split,
-            eval_split=eval_split,
-            **kwargs)
+        super().__init__(**kwargs)
+
+        self.category_map = {k: v[-1] for k, v in SUBJECT_MAPPING.items()}
 
     def load_from_disk(self, dataset_name_or_path, subset_list, work_dir, **kwargs) -> dict:
         data_dict = {}
@@ -175,11 +168,11 @@ class MMLUAdapter(DataAdapter):
             data_dict[subset_name] = {}
 
             for split_name in [self.train_split, self.eval_split]:
-                if self.train_split == 'train':
+                if split_name == 'train':
                     split_name_suffix = 'dev'
-                elif self.eval_split == 'test':
+                elif split_name == 'test':
                     split_name_suffix = 'test'
-                elif self.eval_split == 'validation':
+                elif split_name == 'validation':
                     split_name_suffix = 'val'
                 else:
                     raise ValueError(f'Invalid split name: {split_name}')
@@ -225,7 +218,7 @@ class MMLUAdapter(DataAdapter):
             'target': 'A'}
 
         Returns:
-            {'data': [(context, continuation), ...]}
+            {'data': [full_prompt], 'multi_choices': self.choices}
 
         """
         prompt = 'The following are multiple choice questions (with answers) about {}.\n\n'.format(
@@ -238,13 +231,13 @@ class MMLUAdapter(DataAdapter):
 
         full_prompt: str = context.strip() + self._generate_prompt(input_d=input_d, include_answer=False)
 
-        return {'data': [full_prompt], 'multi_choices': self.choices}
+        return {'data': [full_prompt], 'multi_choices': self.choices, 'system_prompt': self.prompt_template}
 
     def get_gold_answer(self, input_d: dict) -> str:
         # Get the gold choice
         return input_d.get('target', '')
 
-    def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = 'checkpoint') -> str:
+    def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = EvalType.CHECKPOINT) -> str:
         """
         Parse the model output to get the answer. Could be the best choice index.
 
@@ -256,109 +249,18 @@ class MMLUAdapter(DataAdapter):
         Returns:
             The parsed answer. Depending on the dataset. Usually a string for chat.
         """
-        if eval_type == 'checkpoint':
+        if eval_type == EvalType.CHECKPOINT:
             return result
-        elif eval_type == 'service':
-            return ResponseParser.parse_first_option_with_choices(result, self.choices)  # TODO: to be checked !
-        elif eval_type == 'custom':
-            return ResponseParser.parse_first_option_with_choices(result, self.choices)  # TODO: to be checked !
+        elif eval_type == EvalType.SERVICE:
+            return ResponseParser.parse_first_option_with_choices(result, self.choices)
+        elif eval_type == EvalType.CUSTOM:
+            return ResponseParser.parse_first_option_with_choices(result, self.choices)
         else:
             raise ValueError(f'Invalid eval_type: {eval_type}')
 
     def match(self, gold: str, pred: str) -> float:
         return exact_match(gold=gold, pred=pred)
 
-    def compute_metric(self, review_res_list: list) -> float:
-        """
-        Compute evaluation result by specific metric.
-
-        Args:
-            review_res_list: review score list, e.g. [0, 1, 1, 0, ...]
-
-        Returns:
-            The metric score.
-        """
-        items = [(score, 1.0) for score in review_res_list]
-        return weighted_mean(items)
-
-    def gen_report(self, subset_score_map: dict, report_name: str = None) -> dict:
-        """
-        Generate report for the evaluation.
-
-        Args:
-            subset_score_map: The subset-score mapping. e.g. {subset_name: (score, num), ...}
-            report_name: The user-defined report name.
-
-        Returns:
-        {
-            "name":"MMLU",
-            "metric":"WeightedAverageAccuracy",
-            "score":0.3389,
-            "category":[
-                {
-                    "name":"STEM",
-                    "score":0.2528,
-                    "subset":[
-                        {
-                            "name":"computer_network",
-                            "score":0.2632
-                        },
-                        {
-                            "name":"operating_system",
-                            "score":0.3157
-                        },
-                        {
-                            "name":"computer_architecture",
-                            "score":0.4285
-                        }
-                    ]
-                }
-            ],
-            "total_num":59
-        }
-        """
-        total_num: int = sum([num for _, num in subset_score_map.values()])
-        weighted_avg_acc: float = sum([score * num for score, num in subset_score_map.values()]) / total_num
-        weighted_avg_acc = normalize_score(score=weighted_avg_acc)
-
-        # Get domain-subject mapping
-        subject_review_map = {}
-        for subset_name, (subset_score, num) in subset_score_map.items():
-            domain_name: str = SUBJECT_MAPPING.get(subset_name)[2] if SUBJECT_MAPPING.get(subset_name) else subset_name
-            if domain_name in subject_review_map:
-                subject_review_map[domain_name].append((subset_name, subset_score, num))
-            else:
-                subject_review_map[domain_name] = [(subset_name, subset_score, num)]
-
-        # Get domain score
-        category_list = []
-        for domain_name, domain_res_list in subject_review_map.items():
-            domain_weighted_avg_acc = sum([score * num for _, score, num in domain_res_list]) / \
-                                      sum([num for _, _, num in domain_res_list])
-            domain_weighted_avg_acc = normalize_score(score=domain_weighted_avg_acc)
-            category_list.append({
-                'name':
-                domain_name,
-                'score':
-                domain_weighted_avg_acc,
-                'subset': [{
-                    'name': subset_name,
-                    'score': normalize_score(score=subset_score)
-                } for subset_name, subset_score, _ in domain_res_list]
-            })
-
-        category_list = sorted(category_list, key=lambda x: x['name'])
-
-        # Get final dict of report
-        res_map = dict(
-            name=report_name or 'mmlu',
-            metric=self.metric_list[0]['name'],
-            score=weighted_avg_acc,
-            category=category_list,
-            total_num=total_num)
-
-        return res_map
-
     @classmethod
     def _generate_prompt(cls, input_d: dict, include_answer=True) -> str:
 

evalscope/benchmarks/mmlu_pro/mmlu_pro_adapter.py

@@ -0,0 +1,110 @@
+from collections import defaultdict
+from typing import Any, Dict
+
+from evalscope.benchmarks import Benchmark, DataAdapter
+from evalscope.constants import AnswerKeys, EvalType
+from evalscope.metrics import AverageAccuracy, exact_match
+from evalscope.models import ChatGenerationModelAdapter
+from evalscope.utils.utils import ResponseParser
+
+
+@Benchmark.register(
+    name='mmlu_pro',
+    dataset_id='modelscope/mmlu-pro',
+    model_adapter=ChatGenerationModelAdapter,
+    subset_list=['default'],
+    metric_list=[AverageAccuracy],
+    few_shot_num=5,
+    train_split='validation',
+    eval_split='test',
+    prompt_template=
+    'You are an knowledge expert, you are supposed to answer the multi-choice question to derive your final answer as `The answer is ...`.',  # noqa: E501
+)
+class MMLUProAdapter(DataAdapter):
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+        self.choices = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J']
+        self.categories = [
+            'computer science', 'math', 'chemistry', 'engineering', 'law', 'biology', 'health', 'physics', 'business',
+            'philosophy', 'economics', 'other', 'psychology', 'history'
+        ]
+
+    def gen_prompts(self, data_dict: dict, **kwargs) -> Dict[str, list]:
+        """
+        Generate model prompt from raw input, unify the prompt format for MMLU-Pro benchmark.
+        Return a dict with category as key and list of prompts as value.
+        """
+
+        data_dict = data_dict[self.subset_list[0]]  # Only one subset for MMLU-Pro
+        fewshot_prompts = self.get_fewshot_examples(data_dict)
+
+        #  Use the category as key to group the prompts
+        res_dict = defaultdict(list)
+        # generate prompts for each test sample
+        for entry in data_dict[self.eval_split]:
+            prefix = fewshot_prompts[entry['category']]
+            query = prefix + 'Q: ' + entry['question'] + '\n' + \
+                self.__form_options(entry['options']) + '\n'
+
+            prompt_d = {'data': [query], 'system_prompt': self.prompt_template, AnswerKeys.RAW_INPUT: entry}
+
+            res_dict[entry['category']].append(prompt_d)
+        return res_dict
+
+    def get_fewshot_examples(self, data_dict: dict):
+        # load 5-shot prompts for each category
+        prompts = {c: '' for c in self.categories}
+        for d in data_dict[self.train_split]:
+            prompts[d['category']] += 'Q:' + ' ' + d['question'] + '\n' + \
+                self.__form_options(d['options']) + '\n' + \
+                d['cot_content'] + '\n\n'
+        return prompts
+
+    def __form_options(self, options: list):
+        option_str = 'Options are:\n'
+        for opt, choice in zip(options, self.choices):
+            option_str += f'({choice}): {opt}' + '\n'
+        return option_str
+
+    def get_gold_answer(self, input_d: dict) -> str:
+        """
+        Parse the raw input labels (gold).
+
+        Args:
+            input_d: input raw data. Depending on the dataset.
+
+        Returns:
+            The parsed input. e.g. gold answer ... Depending on the dataset.
+        """
+        return input_d['answer']
+
+    def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = EvalType.CHECKPOINT) -> str:
+        """
+        Parse the predicted result and extract proper answer.
+
+        Args:
+            result: Predicted answer from the model. Usually a string for chat.
+            raw_input_d: The raw input. Depending on the dataset.
+            eval_type: 'checkpoint' or 'service' or `custom`, default: 'checkpoint'
+
+        Returns:
+            The parsed answer. Depending on the dataset. Usually a string for chat.
+        """
+        return ResponseParser.parse_first_option(result)
+
+    def match(self, gold: str, pred: str) -> float:
+        """
+        Match the gold answer and the predicted answer.
+
+        Args:
+            gold (Any): The golden answer. Usually a string for chat/multiple-choice-questions.
+                        e.g. 'A', extracted from get_gold_answer method.
+            pred (Any): The predicted answer. Usually a string for chat/multiple-choice-questions.
+                        e.g. 'B', extracted from parse_pred_result method.
+
+        Returns:
+            The match result. Usually a score (float) for chat/multiple-choice-questions.
+        """
+        return exact_match(gold=gold, pred=pred)

evalscope/benchmarks/race/__init__.py

@@ -1,6 +1 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
-
-from evalscope.benchmarks.race.race_adapter import DATASET_ID, SUBJECT_MAPPING, SUBSET_LIST
-from evalscope.benchmarks.race.race_adapter import RACEAdapter
-from evalscope.benchmarks.race.race_adapter import RACEAdapter as DataAdapterClass
-from evalscope.models.model_adapter import MultiChoiceModelAdapter as ModelAdapterClass  # noqa