evalscope 0.9.0__py3-none-any.whl → 0.10.0__py3-none-any.whl

evalscope/benchmarks/ifeval/utils.py

@@ -0,0 +1,134 @@
+import dataclasses
+from typing import Dict, Optional, Union
+
+from evalscope.benchmarks.ifeval import instructions_registry
+
+
+@dataclasses.dataclass
+class InputExample:
+    key: int
+    instruction_id_list: list[str]
+    prompt: str
+    kwargs: list[Dict[str, Optional[Union[str, int]]]]
+
+
+@dataclasses.dataclass
+class OutputExample:
+    instruction_id_list: list[str]
+    prompt: str
+    response: str
+    follow_all_instructions: bool
+    follow_instruction_list: list[bool]
+
+
+def test_instruction_following_strict(
+    inp,
+    response,
+):
+    """Tests response to see if instructions are followed."""
+    instruction_list = inp.instruction_id_list
+    is_following_list = []
+
+    for index, instruction_id in enumerate(instruction_list):
+        instruction_cls = instructions_registry.INSTRUCTION_DICT[instruction_id]
+        instruction = instruction_cls(instruction_id)
+
+        # Remove None values from kwargs to avoid unexpected keyword argument errors in build_description method.
+        kwargs = {k: v for k, v in inp.kwargs[index].items() if v}
+        instruction.build_description(**kwargs)
+        args = instruction.get_instruction_args()
+        if args and 'prompt' in args:
+            instruction.build_description(prompt=inp.prompt)
+
+        if response.strip() and instruction.check_following(response):
+            is_following_list.append(True)
+        else:
+            is_following_list.append(False)
+
+    return OutputExample(
+        instruction_id_list=inp.instruction_id_list,
+        prompt=inp.prompt,
+        response=response,
+        follow_all_instructions=all(is_following_list),
+        follow_instruction_list=is_following_list,
+    )
+
+
+def test_instruction_following_loose(
+    inp,
+    response,
+):
+    """Tests response for an upper bound for following instructions."""
+    r = response.split('\n')
+    response_remove_first = '\n'.join(r[1:]).strip()
+    response_remove_last = '\n'.join(r[:-1]).strip()
+    response_remove_both = '\n'.join(r[1:-1]).strip()
+    revised_response = response.replace('*', '')
+    revised_response_remove_first = response_remove_first.replace('*', '')
+    revised_response_remove_last = response_remove_last.replace('*', '')
+    revised_response_remove_both = response_remove_both.replace('*', '')
+    all_responses = [
+        response,
+        revised_response,
+        response_remove_first,
+        response_remove_last,
+        response_remove_both,
+        revised_response_remove_first,
+        revised_response_remove_last,
+        revised_response_remove_both,
+    ]
+    instruction_list = inp.instruction_id_list
+    is_following_list = []
+
+    for index, instruction_id in enumerate(instruction_list):
+        instruction_cls = instructions_registry.INSTRUCTION_DICT[instruction_id]
+        instruction = instruction_cls(instruction_id)
+
+        # Remove None values from kwargs to avoid unexpected keyword argument errors in build_description method.
+        kwargs = {k: v for k, v in inp.kwargs[index].items() if v}
+        instruction.build_description(**kwargs)
+        args = instruction.get_instruction_args()
+        if args and 'prompt' in args:
+            instruction.build_description(prompt=inp.prompt)
+
+        is_following = False
+        for r in all_responses:
+            if r.strip() and instruction.check_following(r):
+                is_following = True
+                break
+
+        is_following_list.append(is_following)
+
+    return OutputExample(
+        instruction_id_list=inp.instruction_id_list,
+        prompt=inp.prompt,
+        response=response,
+        follow_all_instructions=all(is_following_list),
+        follow_instruction_list=is_following_list,
+    )
+
+
+def process_results(doc, results):
+    inp = InputExample(
+        key=doc['key'],
+        instruction_id_list=doc['instruction_id_list'],
+        prompt=doc['prompt'],
+        kwargs=doc['kwargs'],
+    )
+    response = results[0]
+
+    out_strict = test_instruction_following_strict(inp, response)
+    out_loose = test_instruction_following_loose(inp, response)
+
+    return {
+        'prompt_level_strict_acc': out_strict.follow_all_instructions,
+        'inst_level_strict_acc': out_strict.follow_instruction_list,
+        'prompt_level_loose_acc': out_loose.follow_all_instructions,
+        'inst_level_loose_acc': out_loose.follow_instruction_list,
+    }
+
+
+def agg_inst_level_acc(items):
+    flat_items = [item for sublist in items for item in sublist]
+    inst_level_acc = sum(flat_items) / len(flat_items)
+    return inst_level_acc

evalscope/benchmarks/iquiz/iquiz_adapter.py

@@ -0,0 +1,63 @@
+from evalscope.benchmarks import Benchmark, DataAdapter
+from evalscope.constants import AnswerKeys, EvalType
+from evalscope.metrics import AverageAccuracy, exact_match
+from evalscope.models import ChatGenerationModelAdapter
+from evalscope.utils.utils import ResponseParser
+
+
+@Benchmark.register(
+    name='iquiz',
+    dataset_id='AI-ModelScope/IQuiz',
+    model_adapter=ChatGenerationModelAdapter,
+    subset_list=['IQ', 'EQ'],
+    metric_list=[AverageAccuracy],
+    few_shot_num=0,
+    train_split=None,
+    eval_split='test',
+    prompt_template='你是一个高智商和高情商的专家，你被要求回答一个选择题，并选出一个正确的选项，解释原因，最终输出格式为：`答案是(选项)`。',  # noqa: E501
+)
+class IQuizAdapter(DataAdapter):
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+        self.choices = ['A', 'B', 'C', 'D', 'E']
+
+    def gen_prompt(self, input_d: dict, subset_name: str, few_shot_list: list, **kwargs) -> dict:
+        """
+        Generate model prompt from input data.
+        example:
+        {
+            "question":"天气预报说本周星期三会下雨，昨天果然下雨了，今天星期几？",
+            "choices":["星期一","星期二","星期三","星期四"],
+            "answer":"D",
+            "level":1
+        }
+        """
+        prompt = f"问题: {input_d['question']}\n"
+        prompt += self.__form_options(input_d['choices'])
+        return {'data': [prompt], 'multi_choices': self.choices, 'system_prompt': self.prompt_template}
+
+    def __form_options(self, options: list):
+        option_str = '选项:\n'
+        for opt, choice in zip(options, self.choices):
+            option_str += f'({choice}): {opt}' + '\n'
+        return option_str
+
+    def get_gold_answer(self, input_d: dict) -> str:
+        """
+        Parse the raw input labels (gold).
+        """
+        return input_d['answer']
+
+    def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = EvalType.CHECKPOINT) -> str:
+        """
+        Parse the predicted result and extract proper answer.
+        """
+        return ResponseParser.parse_first_option_with_choices(result, self.choices)
+
+    def match(self, gold: str, pred: str) -> float:
+        """
+        Match the gold answer and the predicted answer.
+        """
+        return exact_match(gold=gold, pred=pred)

evalscope/benchmarks/mmlu/mmlu_adapter.py

@@ -4,7 +4,7 @@ import os
 
 from evalscope.benchmarks import Benchmark, DataAdapter
 from evalscope.constants import EvalType
-from evalscope.metrics import WeightedAverageAccuracy, exact_match
+from evalscope.metrics import AverageAccuracy, exact_match
 from evalscope.models import MultiChoiceModelAdapter
 from evalscope.utils import ResponseParser, normalize_score
 from evalscope.utils.logger import get_logger
@@ -141,7 +141,7 @@ SUBJECT_MAPPING = {
     dataset_id='modelscope/mmlu',
     model_adapter=MultiChoiceModelAdapter,
     subset_list=SUBSET_LIST,
-    metric_list=[WeightedAverageAccuracy],
+    metric_list=[AverageAccuracy],
     few_shot_num=5,
     train_split='train',
     eval_split='test',
@@ -160,17 +160,19 @@ class MMLUAdapter(DataAdapter):
 
         super().__init__(**kwargs)
 
+        self.category_map = {k: v[-1] for k, v in SUBJECT_MAPPING.items()}
+
     def load_from_disk(self, dataset_name_or_path, subset_list, work_dir, **kwargs) -> dict:
         data_dict = {}
         for subset_name in subset_list:
             data_dict[subset_name] = {}
 
             for split_name in [self.train_split, self.eval_split]:
-                if self.train_split == 'train':
+                if split_name == 'train':
                     split_name_suffix = 'dev'
-                elif self.eval_split == 'test':
+                elif split_name == 'test':
                     split_name_suffix = 'test'
-                elif self.eval_split == 'validation':
+                elif split_name == 'validation':
                     split_name_suffix = 'val'
                 else:
                     raise ValueError(f'Invalid split name: {split_name}')
@@ -229,7 +231,7 @@ class MMLUAdapter(DataAdapter):
 
         full_prompt: str = context.strip() + self._generate_prompt(input_d=input_d, include_answer=False)
 
-        return {'data': [full_prompt], 'multi_choices': self.choices}
+        return {'data': [full_prompt], 'multi_choices': self.choices, 'system_prompt': self.prompt_template}
 
     def get_gold_answer(self, input_d: dict) -> str:
         # Get the gold choice
@@ -259,84 +261,6 @@ class MMLUAdapter(DataAdapter):
     def match(self, gold: str, pred: str) -> float:
         return exact_match(gold=gold, pred=pred)
 
-    def gen_report(self, subset_score_map: dict, report_name: str = None) -> dict:
-        """
-        Generate report for the evaluation.
-
-        Args:
-            subset_score_map: The subset-score mapping. e.g. {subset_name: (score, num), ...}
-            report_name: The user-defined report name.
-
-        Returns:
-        {
-            "name":"MMLU",
-            "metric":"WeightedAverageAccuracy",
-            "score":0.3389,
-            "category":[
-                {
-                    "name":"STEM",
-                    "score":0.2528,
-                    "subset":[
-                        {
-                            "name":"computer_network",
-                            "score":0.2632
-                        },
-                        {
-                            "name":"operating_system",
-                            "score":0.3157
-                        },
-                        {
-                            "name":"computer_architecture",
-                            "score":0.4285
-                        }
-                    ]
-                }
-            ],
-            "total_num":59
-        }
-        """
-        total_num: int = sum([num for _, num in subset_score_map.values()])
-        weighted_avg_acc: float = sum([score * num for score, num in subset_score_map.values()]) / total_num
-        weighted_avg_acc = normalize_score(score=weighted_avg_acc)
-
-        # Get domain-subject mapping
-        subject_review_map = {}
-        for subset_name, (subset_score, num) in subset_score_map.items():
-            domain_name: str = SUBJECT_MAPPING.get(subset_name)[2] if SUBJECT_MAPPING.get(subset_name) else subset_name
-            if domain_name in subject_review_map:
-                subject_review_map[domain_name].append((subset_name, subset_score, num))
-            else:
-                subject_review_map[domain_name] = [(subset_name, subset_score, num)]
-
-        # Get domain score
-        category_list = []
-        for domain_name, domain_res_list in subject_review_map.items():
-            domain_weighted_avg_acc = sum([score * num for _, score, num in domain_res_list]) / \
-                                      sum([num for _, _, num in domain_res_list])
-            domain_weighted_avg_acc = normalize_score(score=domain_weighted_avg_acc)
-            category_list.append({
-                'name':
-                domain_name,
-                'score':
-                domain_weighted_avg_acc,
-                'subset': [{
-                    'name': subset_name,
-                    'score': normalize_score(score=subset_score)
-                } for subset_name, subset_score, _ in domain_res_list]
-            })
-
-        category_list = sorted(category_list, key=lambda x: x['name'])
-
-        # Get final dict of report
-        res_map = dict(
-            name=report_name or 'mmlu',
-            metric=self.metric_list[0]['name'],
-            score=weighted_avg_acc,
-            category=category_list,
-            total_num=total_num)
-
-        return res_map
-
     @classmethod
     def _generate_prompt(cls, input_d: dict, include_answer=True) -> str:
 

evalscope/benchmarks/mmlu_pro/mmlu_pro_adapter.py

@@ -3,7 +3,7 @@ from typing import Any, Dict
 
 from evalscope.benchmarks import Benchmark, DataAdapter
 from evalscope.constants import AnswerKeys, EvalType
-from evalscope.metrics import WeightedAverageAccuracy, exact_match
+from evalscope.metrics import AverageAccuracy, exact_match
 from evalscope.models import ChatGenerationModelAdapter
 from evalscope.utils.utils import ResponseParser
 
@@ -13,7 +13,7 @@ from evalscope.utils.utils import ResponseParser
     dataset_id='modelscope/mmlu-pro',
     model_adapter=ChatGenerationModelAdapter,
     subset_list=['default'],
-    metric_list=[WeightedAverageAccuracy],
+    metric_list=[AverageAccuracy],
     few_shot_num=5,
     train_split='validation',
     eval_split='test',

evalscope/benchmarks/race/race_adapter.py

@@ -4,9 +4,9 @@ import os
 
 from evalscope.benchmarks import Benchmark, DataAdapter
 from evalscope.constants import EvalType
-from evalscope.metrics import WeightedAverageAccuracy, exact_match
+from evalscope.metrics import AverageAccuracy, exact_match
 from evalscope.models import MultiChoiceModelAdapter
-from evalscope.utils import ResponseParser, normalize_score
+from evalscope.utils import ResponseParser
 from evalscope.utils.io_utils import jsonl_to_list
 from evalscope.utils.logger import get_logger
 
@@ -14,15 +14,13 @@ from evalscope.utils.logger import get_logger
 
 logger = get_logger()
 
-SUBJECT_MAPPING = {'high': 'High', 'middle': 'Middle'}
-
 
 @Benchmark.register(
     name='race',
     dataset_id='modelscope/race',
     model_adapter=MultiChoiceModelAdapter,
     subset_list=['high', 'middle'],
-    metric_list=[WeightedAverageAccuracy],
+    metric_list=[AverageAccuracy],
     few_shot_num=3,
     train_split='train',
     eval_split='test',
@@ -84,7 +82,7 @@ class RACEAdapter(DataAdapter):
 
         full_prompt: str = context.strip() + self._generate_prompt(input_d=input_d, include_answer=False)
 
-        return {'data': [full_prompt], 'multi_choices': self.choices}
+        return {'data': [full_prompt], 'multi_choices': self.choices, 'system_prompt': self.prompt_template}
 
     def get_gold_answer(self, input_d: dict) -> str:
         # Get the gold choice
@@ -114,73 +112,6 @@ class RACEAdapter(DataAdapter):
     def match(self, gold: str, pred: str) -> float:
         return exact_match(gold=gold, pred=pred)
 
-    def gen_report(self, subset_score_map: dict, report_name: str = None) -> dict:
-        """
-        Generate report for the evaluation.
-
-        Args:
-            subset_score_map: The subset-score mapping. e.g. {subset_name: (score, num), ...}
-            report_name: The user-defined report name.
-
-        Returns:
-        {
-            "name":"RACE",
-            "metric":"WeightedAverageAccuracy",
-            "score":0.3389,
-            "category":[
-                {
-                   "name":"High",
-                   "score":0.2528,
-                   "subset":[
-                       {
-                           "name":"high",
-                           "score":0.2528
-                       }
-                   ]
-                }
-            ],
-            "total_num":59
-        }
-        """
-        total_num: int = sum([num for _, num in subset_score_map.values()])
-        weighted_avg_acc: float = sum([score * num for score, num in subset_score_map.values()]) / total_num
-
-        # Get domain-subject mapping
-        subject_review_map = {}
-        for subset_name, (subset_score, num) in subset_score_map.items():
-            domain_name: str = SUBJECT_MAPPING.get(subset_name)
-            if domain_name in subject_review_map:
-                subject_review_map[domain_name].append((subset_name, subset_score, num))
-            else:
-                subject_review_map[domain_name] = [(subset_name, subset_score, num)]
-
-        # Get domain score
-        category_list = []
-        for domain_name, domain_res_list in subject_review_map.items():
-            domain_weighted_avg_acc = sum([score * num for _, score, num in domain_res_list]) / \
-                                     sum([num for _, _, num in domain_res_list])
-            domain_weighted_avg_acc = normalize_score(score=domain_weighted_avg_acc)
-            category_list.append({
-                'name':
-                domain_name,
-                'score':
-                normalize_score(score=domain_weighted_avg_acc),
-                'subset': [{
-                    'name': subset_name,
-                    'score': subset_score
-                } for subset_name, subset_score, _ in domain_res_list]
-            })
-
-        # Get final dict of report
-        res_map = dict(
-            name=report_name or 'race',
-            metric=self.metric_list[0]['name'],
-            score=weighted_avg_acc,
-            category=category_list,
-            total_num=total_num)
-
-        return res_map
-
     @classmethod
     def _generate_prompt(cls, input_d: dict, include_answer=True) -> str:
 

evalscope/benchmarks/trivia_qa/trivia_qa_adapter.py

@@ -1,15 +1,12 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 # Copyright (c) EleutherAI Inc, and its affiliates.
 import csv
-import numpy as np
 import os
-from typing import List
 
 from evalscope.benchmarks import Benchmark
 from evalscope.benchmarks.data_adapter import DataAdapter
 from evalscope.constants import EvalType
-from evalscope.metrics import WeightedAverageAccuracy
-from evalscope.metrics.metrics import exact_match
+from evalscope.metrics import AverageAccuracy
 from evalscope.models import ChatGenerationModelAdapter
 from evalscope.utils import get_logger
 
@@ -23,7 +20,7 @@ logger = get_logger()
     dataset_id='modelscope/trivia_qa',
     model_adapter=ChatGenerationModelAdapter,
     subset_list=['default'],
-    metric_list=[WeightedAverageAccuracy],
+    metric_list=[AverageAccuracy],
     few_shot_num=5,
     train_split='dev',
     eval_split='test',
@@ -104,7 +101,7 @@ class TriviaQaAdapter(DataAdapter):
         context += self._generate_prompt(input_d=input_d, include_answer=False)
         full_prompt = context
 
-        return {'data': [full_prompt], 'system_prompt': prompt}
+        return {'data': [full_prompt], 'system_prompt': prompt or self.prompt_template}
 
     def get_gold_answer(self, input_d: dict) -> list:
         # Get the gold choice

evalscope/benchmarks/truthful_qa/truthful_qa_adapter.py

@@ -9,8 +9,7 @@ from typing import List
 from evalscope.benchmarks import Benchmark
 from evalscope.benchmarks.data_adapter import DataAdapter
 from evalscope.constants import EvalType
-from evalscope.metrics import WeightedAverageAccuracy
-from evalscope.metrics.metrics import weighted_mean
+from evalscope.metrics import AverageAccuracy
 from evalscope.models import ContinuationLogitsModelAdapter
 from evalscope.utils import get_logger, normalize_score
 
@@ -26,7 +25,7 @@ logger = get_logger()
     dataset_id='modelscope/truthful_qa',
     model_adapter=ContinuationLogitsModelAdapter,
     subset_list=['multiple_choice'],
-    metric_list=[WeightedAverageAccuracy],
+    metric_list=[AverageAccuracy],
     few_shot_num=0,
     train_split=None,
     eval_split='validation',
@@ -260,7 +259,7 @@ class TruthfulQaAdapter(DataAdapter):
 
         return {'multiple_choice': {'mc1': mc1(mc1_lls), 'mc2': mc2(mc2_lls)}}  # or {'generation': xxx}
 
-    def compute_metric(self, review_res_list: List[dict]) -> float:
+    def compute_metric(self, review_res_list: List[dict]) -> List[dict]:
         """
         Compute evaluation result by specific metric for each subset.
 
@@ -285,56 +284,8 @@ class TruthfulQaAdapter(DataAdapter):
                 logger.error(f'** Unknown review_res: {review_res_d}')
 
         # To get mc2 score
-        items = [(score, 1.0) for score in mc2_list]
-        return weighted_mean(items)
-
-    def gen_report(self, subset_score_map: dict, report_name: str = None) -> dict:
-        """
-        Generate the report for the model output.
-
-        Args:
-            subset_score_map: {subset_name: (score, num), ...}
-            report_name: The user-defined report name.
-
-        Returns:
-        {
-            "name":"TruthfulQA",
-            "metric":"WeightedAverageAccuracy",
-            "score":0.3389,
-            "category":[
-                {
-                    "name":"DEFAULT",
-                    "score":0.2527,
-                    "subset":[
-                        {
-                            "name":"multiple_choice",
-                            "score":0.3157
-                        },
-                        # {
-                        #     "name":"generation",
-                        #     "score":0.2631
-                        # }
-                    ]
-                }
-            ],
-            "total_num":100
-        }
-        """
-        total_num: int = sum([num for _, num in subset_score_map.values()])
-        weighted_avg_acc: float = sum([score * num for score, num in subset_score_map.values()]) / total_num
-        weighted_avg_acc = normalize_score(score=weighted_avg_acc)
-        cate_avg_list = [{
-            'name': subset_name,
-            'score': normalize_score(score=score)
-        } for subset_name, (score, _) in subset_score_map.items()]
-
-        category_d = dict(name='DEFAULT', score=weighted_avg_acc, subset=cate_avg_list)
-
-        res_map = dict(
-            name=report_name or 'truthful_qa',
-            metric=self.metric_list[0]['name'],
-            score=weighted_avg_acc,
-            category=[category_d],
-            total_num=total_num)
-
-        return res_map
+        return [{
+            'metric_name': self.metric_list[0].name,
+            'score': self.metric_list[0].object(mc2_list),
+            'num': len(mc2_list)
+        }]

evalscope/cli/cli.py

@@ -2,6 +2,7 @@
 
 import argparse
 
+from evalscope.cli.start_app import StartAppCMD
 from evalscope.cli.start_eval import EvalCMD
 from evalscope.cli.start_perf import PerfBenchCMD
 
@@ -12,6 +13,7 @@ def run_cmd():
 
     PerfBenchCMD.define_args(subparsers)
     EvalCMD.define_args(subparsers)
+    StartAppCMD.define_args(subparsers)
 
     args = parser.parse_args()
 

evalscope/cli/start_app.py

@@ -0,0 +1,29 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import os
+from argparse import ArgumentParser
+
+from evalscope.cli.base import CLICommand
+from evalscope.report.app import create_app
+
+
+def subparser_func(args):
+    """ Function which will be called for a specific sub parser.
+    """
+    return StartAppCMD(args)
+
+
+class StartAppCMD(CLICommand):
+    name = 'app'
+
+    def __init__(self, args):
+        self.args = args
+
+    @staticmethod
+    def define_args(parsers: ArgumentParser):
+        """ define args for create pipeline template command.
+        """
+        parser = parsers.add_parser(StartAppCMD.name)
+        parser.set_defaults(func=subparser_func)
+
+    def execute(self):
+        create_app()