evalscope 0.8.2__py3-none-any.whl → 0.10.0__py3-none-any.whl

evalscope/benchmarks/data_adapter.py

@@ -2,10 +2,11 @@
 import os.path
 import random
 from abc import ABC, abstractmethod
-from typing import Any, Optional
+from typing import Any, List, Optional
 
-from evalscope.benchmarks import Benchmark
-from evalscope.constants import DEFAULT_DATASET_CACHE_DIR, AnswerKeys, HubType
+from evalscope.constants import DEFAULT_DATASET_CACHE_DIR, AnswerKeys, EvalType, HubType
+from evalscope.metrics import Metric
+from evalscope.report import Report, ReportGenerator
 from evalscope.utils.logger import get_logger
 
 logger = get_logger()
@@ -14,15 +15,22 @@ logger = get_logger()
 class DataAdapter(ABC):
 
     def __init__(self,
+                 name: str,
                  subset_list: list,
-                 metric_list: list,
+                 metric_list: List[Metric],
                  few_shot_num: Optional[int] = 0,
                  train_split: Optional[str] = None,
                  eval_split: Optional[str] = None,
-                 prompt_template: str = '',
+                 prompt_template: Optional[str] = None,
                  **kwargs):
         """
+        Data Adapter for the benchmark. You need to implement the following methods:
+            - gen_prompt
+            - get_gold_answer
+            - parse_pred_result
+            - match
         Args:
+            name: str, the name of the benchmark.
             subset_list: list of subset names for the dataset.
             metric_list: list, the metric list to evaluate the model on specific benchmark.
             few_shot_num: int, number of few-shot examples. Default: 0
@@ -32,6 +40,7 @@ class DataAdapter(ABC):
                 e.g. for ARC, it is `The following are multiple choice questions, please output correct answer in
                     the form of A or B or C or D, do not output explanation:`
         """
+        self.name = name
         self.subset_list = subset_list
         self.metric_list = metric_list
         self.few_shot_num = few_shot_num
@@ -39,6 +48,7 @@ class DataAdapter(ABC):
         self.eval_split = eval_split
         self.prompt_template = prompt_template
         self.config_kwargs = kwargs
+        self.category_map = kwargs.get('category_map', {})
 
     def load(self,
              dataset_name_or_path: str,
@@ -55,33 +65,36 @@ class DataAdapter(ABC):
 
         """
         dataset_name_or_path = os.path.expanduser(dataset_name_or_path)
+        subset_list = subset_list or self.subset_list
 
         # Try to load dataset from local disk
         if os.path.exists(dataset_name_or_path):
-            logger.info(
-                f'Loading dataset from local disk: > dataset_name: {dataset_name_or_path}  > work_dir: {work_dir}')
+            logger.info(f'Loading dataset from work_dir: {work_dir}: > dataset_name: {dataset_name_or_path} > \
+                    subsets: {subset_list}')
             data_dict = self.load_from_disk(dataset_name_or_path, subset_list, work_dir, **kwargs)
             if len(data_dict) == 0 or len(next(iter(data_dict.values()))) == 0:
                 raise ValueError(f'Local dataset is empty: {dataset_name_or_path}')
         else:
+            from modelscope.msdatasets import MsDataset
+
             # Load dataset from remote
-            logger.info(f'Loading dataset from {datasets_hub} hub: >dataset_name: {dataset_name_or_path}')
+            logger.info(
+                f'Loading dataset from {datasets_hub}: > dataset_name: {dataset_name_or_path} > subsets: {subset_list}')
             data_dict = {}
             split_list = [split for split in [self.train_split, self.eval_split] if split is not None]
             if len(split_list) == 0:
                 logger.error(f'Got empty split list: {split_list}')
 
-            subset_list = subset_list if subset_list is not None else self.subset_list
             for sub_name in subset_list:
                 data_dict[sub_name] = {}
                 # e.g. train: few-shot, test: target dataset to evaluate
                 for split in split_list:
-                    dataset = Benchmark.load(
+                    dataset = MsDataset.load(
                         dataset_name=dataset_name_or_path,
-                        subset=sub_name,
+                        subset_name=sub_name,
                         split=split,
+                        cache_dir=work_dir,
                         hub=datasets_hub,
-                        work_dir=work_dir,
                         **kwargs)
 
                     data_dict[sub_name].update({split: dataset})
@@ -132,30 +145,105 @@ class DataAdapter(ABC):
                 prompt_d[AnswerKeys.RAW_INPUT] = sample_d
                 res_dict[sub_name].append(prompt_d)
 
-        rnd = random.Random()
-        rnd.seed(42)
-        for k, v in res_dict.items():
-            rnd.shuffle(v)
-
         return res_dict
 
-    @abstractmethod
-    def gen_prompt(self, *args, **kwargs) -> Any:
+    def get_fewshot_examples(self, data_list: list, k: int, few_shot_random: bool = True):
+
+        if k > len(data_list):
+            k = len(data_list)
+        if few_shot_random:
+            return random.sample(data_list, k)
+        else:
+            return data_list[:k]
+
+    def compute_metric(self, review_res_list: list) -> List[dict]:
+        """
+        Compute evaluation result by specific metrics.
+
+        Args:
+            review_res_list: list, the review result list, each item of which is match result for gold and pred.
+
+        Returns:
+            Metric results. e.g. [{'metric_name': 'AverageAccuracy', 'score': 0.3389, 'num': 100}]
+        """
+        if len(self.metric_list) == 0:
+            raise ValueError('No metric list found for the benchmark.')
+
+        res_list = []
+        for metric in self.metric_list:
+            metric_name = metric.name
+            metric_func = metric.object
+            res_list.append({
+                'metric_name': metric_name,
+                'score': metric_func(review_res_list),
+                'num': len(review_res_list)
+            })
+        return res_list
+
+    def gen_report(self, subset_score_map: dict, report_name: str = None, **kwargs) -> Report:
+        """
+        Generate report for the evaluation results for all subsets.
+
+        Args:
+            subset_score_map: The subset-score map.
+                e.g. {subset_name: [{'metric_name': 'AverageAccuracy', 'score': 0.3389, 'num': 100}]}
+
+            report_name: str, the user-defined report name. Default: None
+
+        Returns: The evaluation report.
+
+        Here is a format example for gsm8k:
+        {
+            "name": "qwen2.5_gsm8k",
+            "metrics": [
+                {
+                    "name": "AverageAccuracy",
+                    "categories": [
+                        {
+                            "name": "default",
+                            "subsets": [
+                                {
+                                    "name": "main",
+                                    "score": 0.0,
+                                    "num": 2
+                                }
+                            ],
+                            "num": 2,
+                            "score": 0.0,
+                            "macro_score": 0.0
+                        }
+                    ],
+                    "num": 2,
+                    "score": 0.0,
+                    "macro_score": 0.0
+                }
+            ],
+            "dataset_name": "gsm8k",
+            "model_name": "qwen2.5"
+        }
+        """  # noqa: E501
+        kwargs['category_map'] = self.category_map
+        kwargs['metric_list'] = self.metric_list
+        return ReportGenerator.gen_report(subset_score_map, report_name, **kwargs)
+
+    def gen_prompt(self, input_d: dict, subset_name: str, few_shot_list: list, **kwargs) -> Any:
         """
         Generate model prompt from raw input, unify the prompt format for different datasets.
         The input format is compatible with OpenAI Chat Completions APIs.
-        Refer to: https://platform.openai.com/docs/guides/gpt/chat-completions-api
 
         Args:
             input_d (Any): The raw input. Depending on the dataset.
+            subset_name (str): The subset name.
+            few_shot_list (list): The few-shot examples.
 
         Returns:
+            For class ChatGenerationModelAdapter, the output format is:
+                {'data': [full_prompt], 'system_prompt': (str, optional)},  -- full_prompt: str, the constructed prompt for each sample from dataset.
             For class MultiChoiceModelAdapter, the output format is:
-                {'data': [full_prompt]},  -- full_prompt: str, the constructed prompt for each sample from dataset.
-
+                {'data': [full_prompt], 'multi_choices': self.choices}  -- full_prompt: str, the constructed prompt for each sample from dataset.
             For class ContinuationEvalModelAdapter, the output format is:
-                {'data': ctx_continuation_pair_list, 'multi_choices': self.choices}
-        """
+                {'data': ctx_continuation_pair_list, 'multi_choices': self.choices} -- ctx_continuation_pair_list: list, the context-continuation pair list.
+        """  # noqa: E501
         raise NotImplementedError
 
     @abstractmethod
@@ -172,7 +260,7 @@ class DataAdapter(ABC):
         raise NotImplementedError
 
     @abstractmethod
-    def parse_pred_result(self, result: Any, raw_input_d: dict = None, eval_type: str = 'checkpoint') -> Any:
+    def parse_pred_result(self, result: Any, raw_input_d: dict = None, eval_type: str = EvalType.CHECKPOINT) -> Any:
         """
         Parse the predicted result and extract proper answer.
 
@@ -193,71 +281,11 @@ class DataAdapter(ABC):
 
         Args:
             gold (Any): The golden answer. Usually a string for chat/multiple-choice-questions.
-                        e.g. 'A'
+                        e.g. 'A', extracted from get_gold_answer method.
             pred (Any): The predicted answer. Usually a string for chat/multiple-choice-questions.
-                        e.g. 'B'
+                        e.g. 'B', extracted from parse_pred_result method.
 
         Returns:
             The match result. Usually a score (float) for chat/multiple-choice-questions.
         """
         raise NotImplementedError
-
-    @abstractmethod
-    def compute_metric(self, review_res_list: list) -> Any:
-        """
-        Compute evaluation result by specific metrics.
-
-        Args:
-            review_res_list: list, the review result list, each item of which is match result for gold and pred.
-
-        Attributes:
-            DataAdapter.metric_func_map: metric_name -> metric_func mapping,
-                e.g. {'WeightedAverageAccuracy': weighted_average_acc}
-
-        Returns:
-            Metric results.
-        """
-        raise NotImplementedError
-
-    def gen_report(self, subset_score_map: dict, report_name: str = None) -> dict:
-        """
-        Generate report for the evaluation results for all subsets.
-
-        Args:
-            subset_score_map: The subset-score map.
-                e.g. {subset_name: (score, num)}
-
-            report_name: str, the user-defined report name. Default: None
-
-        Returns: The evaluation report.  Note: should normalize the score by normalize_score method in utils.
-
-        Here is a format example for ARC-Challenge:
-        {
-            "name":"ARC-Challenge",
-            "metric":"WeightedAverageAccuracy",
-            "score": 0.3389,
-            "category":[
-                {
-                    "name":"DEFAULT",
-                    "score": 0.3389,
-                    "subset":[
-                        {
-                            "name":"ARC-Challenge",
-                            "score": 0.3389
-                        },
-                    ]
-                }
-            ],
-            "total_num":100
-        }
-        """
-        raise NotImplementedError
-
-    def get_fewshot_examples(self, data_list: list, k: int, few_shot_random: bool = True):
-
-        if k > len(data_list):
-            k = len(data_list)
-        if few_shot_random:
-            return random.sample(data_list, k)
-        else:
-            return data_list[:k]

evalscope/benchmarks/general_qa/__init__.py

@@ -1,6 +1 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
-
-from evalscope.benchmarks.general_qa.general_qa_adapter import DATASET_ID, SUBSET_LIST
-from evalscope.benchmarks.general_qa.general_qa_adapter import GeneralQAAdapter
-from evalscope.benchmarks.general_qa.general_qa_adapter import GeneralQAAdapter as DataAdapterClass
-from evalscope.models.model_adapter import ChatGenerationModelAdapter as ModelAdapterClass

evalscope/benchmarks/general_qa/general_qa_adapter.py

@@ -1,39 +1,34 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 import glob
-import json
 import os.path
 from collections import defaultdict
-from typing import Any, Optional
+from typing import List
 
-from evalscope.benchmarks.data_adapter import DataAdapter
-from evalscope.metrics.metrics import bleu_ngram_one_sample, weighted_mean
-from evalscope.metrics.rouge_metric import compute_rouge_score_one_sample_zh
+from evalscope.benchmarks import Benchmark, DataAdapter
+from evalscope.metrics import AverageBLEU, bleu_ngram_one_sample, compute_rouge_score_one_sample_zh, mean
+from evalscope.models import ChatGenerationModelAdapter
 from evalscope.utils.io_utils import jsonl_to_list
 from evalscope.utils.logger import get_logger
 
 logger = get_logger()
 
-DATASET_ID = 'general_qa'
-SUBSET_LIST = ['default']
-
 
+@Benchmark.register(
+    name='general_qa',
+    dataset_id='general_qa',
+    model_adapter=ChatGenerationModelAdapter,
+    subset_list=['default'],
+    metric_list=[AverageBLEU],
+    few_shot_num=0,
+    train_split=None,
+    eval_split='test',
+)
 class GeneralQAAdapter(DataAdapter):
     # TODO: set few_shot_num
 
-    def __init__(self,
-                 subset_list: list = None,
-                 metric_list: list = None,
-                 train_split: str = None,
-                 eval_split: str = 'test',
-                 **kwargs):
-        if subset_list is None:
-            subset_list = SUBSET_LIST
-
-        if metric_list is None:
-            metric_list = [{'name': 'WeightedAverageBLEU', 'object': weighted_mean}]
+    def __init__(self, **kwargs):
 
-        super().__init__(
-            subset_list=subset_list, metric_list=metric_list, train_split=train_split, eval_split=eval_split, **kwargs)
+        super().__init__(**kwargs)
 
     def load(self, dataset_name_or_path: str, subset_list: list = None, **kwargs) -> dict:
 
@@ -71,7 +66,7 @@ class GeneralQAAdapter(DataAdapter):
 
         # if len(history) > 0:
         #     prompt = '\n'.join(history) + '\n' + prompt
-        return {'data': [prompt]}
+        return {'data': [prompt], 'system_prompt': self.prompt_template}
 
     def get_gold_answer(self, input_d: dict) -> str:
         """
@@ -95,14 +90,14 @@ class GeneralQAAdapter(DataAdapter):
         """
         return result
 
-    def match(self, gold: str, pred: str) -> float:
+    def match(self, gold: str, pred: str) -> dict:
         """
         Args:
             gold: str
             pred: str
 
         Returns:
-            bleu_score: float
+            bleu_score: dict
 
         """
         res = dict()
@@ -110,10 +105,9 @@ class GeneralQAAdapter(DataAdapter):
         bleu_dict = bleu_ngram_one_sample(pred, gold)
         res.update(rouge_dict)
         res.update(bleu_dict)
-        # return bleu(item)
         return res
 
-    def compute_metric(self, review_res_list: list) -> float:
+    def compute_metric(self, review_res_list: List[dict]) -> List[dict]:
         """
         compute weighted mean of the bleu score of all samples
 
@@ -121,62 +115,12 @@ class GeneralQAAdapter(DataAdapter):
             review_res_list: [score1, score2, ...]
 
         Returns:
-            avg_res: float
+            avg_res: List[dict]
 
         """
         items = defaultdict(list)
         for scores in review_res_list:
             for k, v in scores.items():
-                items[k].append((v, 1.0))
+                items[k].append(v)
         # items = [(score, 1.0) for score in review_res_list]
-        res = {k: weighted_mean(v) for k, v in items.items()}
-        # return weighted_mean(items)
-        return res
-
-    def gen_report(self, subset_score_map: dict, report_name: str = None) -> dict:
-        """
-        Args:
-            subset_score_map: {subset_name: (score_dict, num), ...}
-            report_name: str, the user-defined report name.
-
-        Returns:
-        {
-            "name":"GeneralQA",
-            "metric":"WeightedAverageBLEU",
-            "score":0.399,
-            "category":[
-                {
-                    "name":"DEFAULT",
-                    "score":0.399,
-                    "subset":[
-                        {
-                            "name":"default",
-                            "score":0.399
-                        },
-                    ]
-                }
-            ],
-            "total_num":10
-        }
-        """
-        total_num: int = sum([num for _, num in subset_score_map.values()])
-        # weighted_avg_bleu: float = sum([score * num for score, num in subset_score_map.values()]) / total_num
-        cate_avg_list = [{
-            'name': subset_name,
-            'score': score_dict
-        } for subset_name, (score_dict, _) in subset_score_map.items()]
-        total_avg_list = defaultdict(float)
-        for score_dict, num in subset_score_map.values():
-            for metric, score in score_dict.items():
-                total_avg_list[metric] += score * num / total_num
-
-        category_d = dict(name='DEFAULT', score=total_avg_list, subset=cate_avg_list)
-
-        res_map = dict(
-            name=report_name or 'general_qa',
-            metric=self.metric_list[0]['name'],
-            score=total_avg_list,
-            category=[category_d],
-            total_num=total_num)
-
-        return res_map
+        return [{'metric_name': k, 'score': mean(v), 'num': len(v)} for k, v in items.items()]

evalscope/benchmarks/gsm8k/__init__.py

@@ -1,5 +1 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
-
-from evalscope.benchmarks.gsm8k.gsm8k_adapter import DATASET_ID, SUBSET_LIST
-from evalscope.benchmarks.gsm8k.gsm8k_adapter import GSM8KAdapter as DataAdapterClass
-from evalscope.models.model_adapter import ChatGenerationModelAdapter as ModelAdapterClass  # noqa

evalscope/benchmarks/gsm8k/gsm8k_adapter.py

@@ -1,70 +1,51 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 # Copyright (c) EleutherAI, Inc. and its affiliates.
+# flake8: noqa
 import math
 import os
 import re
 
-from evalscope.benchmarks import DataAdapter
-from evalscope.metrics.metrics import exact_match, weighted_mean
-from evalscope.utils import normalize_score
+from evalscope.benchmarks import Benchmark, DataAdapter
+from evalscope.metrics import AverageAccuracy
+from evalscope.models import ChatGenerationModelAdapter
 from evalscope.utils.io_utils import jsonl_to_list
 from evalscope.utils.logger import get_logger
 
-# flake8: noqa
-
 logger = get_logger()
 
-DATASET_ID = 'modelscope/gsm8k'
-SUBSET_LIST = ['main']
-ANS_RE = re.compile(r'#### (\-?[0-9\.\,]+)')
-INVALID_ANS = '[invalid]'
-
 
+@Benchmark.register(
+    name='gsm8k',
+    dataset_id='modelscope/gsm8k',
+    model_adapter=ChatGenerationModelAdapter,
+    subset_list=['main'],
+    metric_list=[AverageAccuracy],
+    few_shot_num=4,
+    train_split='train',
+    eval_split='test',
+    prompt_template='',
+)
 class GSM8KAdapter(DataAdapter):
 
-    def __init__(self,
-                 subset_list: list = None,
-                 metric_list: list = None,
-                 few_shot_num: int = None,
-                 train_split: str = 'train',
-                 eval_split: str = 'test',
-                 prompt_template: str = '',
-                 **kwargs):
+    def __init__(self, **kwargs):
         """
         Data adapter for GSM8K dataset.
 
         Args:
             subset_list (list): Subset list for the dataset. Default: ['main']
-            metric_list (list): Metric list for the dataset. Default: [{'name': 'WeightedAverageAccuracy', 'object': weighted_mean}]
+            metric_list (list): Metric list for the dataset. Default: [{'name': 'AverageAccuracy', 'object': mean}]
             few_shot_num (int): Number of few-shot examples. Default: 4
             train_split (str): Train split name. Default: 'train'
             eval_split (str): The target eval split name. Default: 'test'
             **kwargs: ...
         """
-
-        if subset_list is None:
-            subset_list = SUBSET_LIST
-
-        if metric_list is None:
-            metric_list = [{'name': 'WeightedAverageAccuracy', 'object': weighted_mean}]
-
-        if few_shot_num is None:
-            logger.info(f'Set 4-shot examples by system for GSM8K.')
-            few_shot_num = 4
-
+        few_shot_num = kwargs.get('few_shot_num', 4)
         if few_shot_num != 4 and few_shot_num != 0:
             logger.error(f'GSM8K uses 4-shot examples with CoT or 0-shot by system, but got {few_shot_num}. '
                          f'Use 4-shot by default.')
-            few_shot_num = 4
+            kwargs['few_shot_num'] = 4
 
-        super().__init__(
-            subset_list=subset_list,
-            metric_list=metric_list,
-            few_shot_num=few_shot_num,
-            train_split=train_split,
-            eval_split=eval_split,
-            prompt_template=prompt_template,
-            **kwargs)
+        super().__init__(**kwargs)
 
     def load_from_disk(self, dataset_name_or_path, subset_list, work_dir, **kwargs) -> dict:
         data_dict = {}
@@ -94,9 +75,8 @@ class GSM8KAdapter(DataAdapter):
         use_fewshot = self.few_shot_num > 0
 
         full_prompt = self._generate_prompt(input_d, few_shot_list=few_shot_list, use_fewshot=use_fewshot)
-        full_prompt = f'{self.prompt_template}\n{full_prompt}' if self.prompt_template else full_prompt
 
-        return {'data': [full_prompt]}
+        return {'data': [full_prompt], 'system_prompt': self.prompt_template}
 
     def get_gold_answer(self, input_d: dict) -> str:
         # Extract the gold answer from the input dict.
@@ -142,66 +122,6 @@ class GSM8KAdapter(DataAdapter):
 
         return number_equal(gold_ans=gold, pred_ans=pred)
 
-    def compute_metric(self, review_res_list: list) -> float:
-        """
-        Compute evaluation result by specific metric.
-
-        Args:
-            review_res_list: review score list, e.g. [0, 1, 1, 0, ...]
-
-        Returns:
-            The metric score.
-        """
-        items = [(score, 1.0) for score in review_res_list]
-        return weighted_mean(items)
-
-    def gen_report(self, subset_score_map: dict, report_name: str = None) -> dict:
-        """
-        Generate the report for the model output.
-
-        Args:
-            subset_score_map: The subset-score mapping. e.g. {subset_name: (score, num), ...}
-            report_name: The user-defined report name. Default: None
-
-        Returns: A dict of metric calculation results. The format is like:
-        {
-            "name":"GSM8K",
-            "metric":"WeightedAverageAccuracy",
-            "score":0.5632,
-            "category":[
-                {
-                    "name":"DEFAULT",
-                    "score":0.5632,
-                    "subset":[
-                        {
-                            "name":"main",
-                            "score":0.5632
-                        },
-                    ]
-                }
-            ],
-            "total_num":100
-        }
-        """
-        total_num: int = sum([num for _, num in subset_score_map.values()])
-        weighted_avg_acc: float = sum([score * num for score, num in subset_score_map.values()]) / total_num
-        weighted_avg_acc = normalize_score(score=weighted_avg_acc)
-        cate_avg_list = [{
-            'name': subset_name,
-            'score': normalize_score(score=score)
-        } for subset_name, (score, _) in subset_score_map.items()]
-
-        category_d = dict(name='DEFAULT', score=weighted_avg_acc, subset=cate_avg_list)
-
-        res_map = dict(
-            name=report_name or 'gsm8k',
-            metric=self.metric_list[0]['name'],
-            score=weighted_avg_acc,
-            category=[category_d],
-            total_num=total_num)
-
-        return res_map
-
     @classmethod
     def _generate_prompt(cls, input_d: dict, few_shot_list: list, use_fewshot: bool = True) -> str:
         if use_fewshot:

evalscope/benchmarks/hellaswag/__init__.py

@@ -1,6 +1 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
-
-from evalscope.benchmarks.hellaswag.hellaswag_adapter import DATASET_ID, SUBSET_LIST
-from evalscope.benchmarks.hellaswag.hellaswag_adapter import HellaSwagAdapter
-from evalscope.benchmarks.hellaswag.hellaswag_adapter import HellaSwagAdapter as DataAdapterClass
-from evalscope.models.model_adapter import ContinuationLogitsModelAdapter as ModelAdapterClass  # noqa