evalscope 0.8.2__py3-none-any.whl → 0.9.0__py3-none-any.whl

evalscope/benchmarks/humaneval/humaneval_adapter.py

@@ -1,38 +1,35 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
-import json
-import os
 import re
-from tqdm import tqdm
 from typing import List
 
-from evalscope.benchmarks.data_adapter import DataAdapter
-from evalscope.metrics.metrics import weighted_mean
-from evalscope.tools.combine_reports import gen_table
-from evalscope.utils import normalize_score
+from evalscope.benchmarks import Benchmark, DataAdapter
+from evalscope.metrics import Pass1
+from evalscope.models import ChatGenerationModelAdapter
 from evalscope.utils.logger import get_logger
 
 logger = get_logger()
 
-DATASET_ID = 'modelscope/humaneval'
-SUBSET_LIST = ['openai_humaneval']
-
 # Example:
 # {"task_id": "HumanEval/0", "prompt": "from typing import List\n\n\ndef has_close_elements(numbers: List[float], threshold: float) -> bool:\n    \"\"\" Check if in given list of numbers, are any two numbers closer to each other than\n    given threshold.\n    >>> has_close_elements([1.0, 2.0, 3.0], 0.5)\n    False\n    >>> has_close_elements([1.0, 2.8, 3.0, 4.0, 5.0, 2.0], 0.3)\n    True\n    \"\"\"\n", "entry_point": "has_close_elements", "canonical_solution": "    for idx, elem in enumerate(numbers):\n        for idx2, elem2 in enumerate(numbers):\n            if idx != idx2:\n                distance = abs(elem - elem2)\n                if distance < threshold:\n                    return True\n\n    return False\n", "test": "\n\nMETADATA = {\n    'author': 'jt',\n    'dataset': 'test'\n}\n\n\ndef check(candidate):\n    assert candidate([1.0, 2.0, 3.9, 4.0, 5.0, 2.2], 0.3) == True\n    assert candidate([1.0, 2.0, 3.9, 4.0, 5.0, 2.2], 0.05) == False\n    assert candidate([1.0, 2.0, 5.9, 4.0, 5.0], 0.95) == True\n    assert candidate([1.0, 2.0, 5.9, 4.0, 5.0], 0.8) == False\n    assert candidate([1.0, 2.0, 3.0, 4.0, 5.0, 2.0], 0.1) == True\n    assert candidate([1.1, 2.2, 3.1, 4.1, 5.1], 1.0) == True\n    assert candidate([1.1, 2.2, 3.1, 4.1, 5.1], 0.5) == False\n\n"}  # noqa
 
 
+@Benchmark.register(
+    name='humaneval',
+    dataset_id='modelscope/humaneval',
+    model_adapter=ChatGenerationModelAdapter,
+    subset_list=['openai_humaneval'],
+    metric_list=[Pass1],
+    few_shot_num=0,
+    train_split=None,
+    eval_split='test',
+    prompt_template='Complete the following python code:\n',
+)
 class HumanevalAdapter(DataAdapter):
     """
     A placeholder for humaneval adapter, see HumanevalEvaluator for implementation.
     """
 
-    def __init__(self,
-                 subset_list: list = None,
-                 metric_list: list = None,
-                 few_shot_num: int = None,
-                 train_split: str = None,
-                 eval_split: str = 'test',
-                 prompt_template: str = 'Complete the following python code:\n',
-                 **kwargs):
+    def __init__(self, **kwargs):
         try:
             from human_eval.data import stream_jsonl, write_jsonl
             from human_eval.evaluation import check_correctness
@@ -41,29 +38,15 @@ class HumanevalAdapter(DataAdapter):
                               'https://github.com/openai/human-eval/tree/master#installation , '
                               'Note that you need to enable the execution code in the human_eval/execution.py first.')
 
-        if subset_list is None:
-            subset_list = SUBSET_LIST
-
-        if metric_list is None:
-            metric_list = [{'name': 'WeightedAverageAccuracy', 'object': weighted_mean}]
-
         self.k = [1]
         self.num_workers = 4
         self.timeout = 4.0
-        self.outputs = kwargs.get('outputs', None)
 
         self.read_problems_func = stream_jsonl
         self.write_jsonl_func = write_jsonl
         self.eval_func = check_correctness
 
-        super().__init__(
-            subset_list=subset_list,
-            metric_list=metric_list,
-            few_shot_num=few_shot_num,
-            train_split=train_split,
-            eval_split=eval_split,
-            prompt_template=prompt_template,
-            **kwargs)
+        super().__init__(**kwargs)
 
     def load_from_disk(self, dataset_name_or_path, subset_list, work_dir, **kwargs) -> dict:
         data_dict = {}
@@ -87,77 +70,6 @@ class HumanevalAdapter(DataAdapter):
 
         return {'data': [full_prompt]}
 
-    def get_answers(self, infer_cfg: dict) -> List[dict]:
-        ans_list: list = []
-        system_prompt: str = ''
-        for task_id, data_d in tqdm(self.problems.items(), total=len(self.problems), desc='Predicting(problems)'):
-            prompt: str = system_prompt + data_d['prompt']
-            inputs: dict = {'data': [prompt]}
-
-            pred_res: dict = self.model_adapter.predict(inputs=inputs, infer_cfg=infer_cfg)
-
-            pred_ans: str = pred_res['choices'][0]['message']['content']
-            pred_ans = self._postprocess(pred_ans)
-
-            ans_list.append({'task_id': task_id, 'completion': pred_ans})
-
-        return ans_list
-
-    def eval(self, infer_cfg: dict, **kwargs):
-
-        # predict
-        ans_list: list = self.get_answers(infer_cfg)
-        ans_out_file: str = os.path.join(self.outputs_structure.predictions_dir, 'human_eval_predictions.jsonl')
-
-        self.write_jsonl_func(filename=ans_out_file, data=ans_list)
-        # logger.info(f'** Dump predictions to {ans_out_file} successfully.')
-        logger.info('** Dump predictions successfully.')
-
-        # evaluate  results: e.g. {'pass@1': 0.333, 'pass@10': 0.111}
-        results = self.eval_func(
-            sample_file=ans_out_file,
-            k=self.k,
-            n_workers=self.num_workers,
-            timeout=self.timeout,
-            problem_file=self.problem_file)
-
-        # output: report
-        report_map: dict = self.gen_report(results=results)
-        report_dir: str = self.outputs_structure.reports_dir
-        report_file: str = os.path.join(report_dir, 'human_eval_report.json')
-
-        with open(report_file, 'w') as f:
-            f.write(json.dumps(report_map, ensure_ascii=False, indent=4))
-        # logger.info(f'** Dump report to {report_file} \n')
-        logger.info('** Dump report \n')
-
-        try:
-            # Make table
-            report_table: str = gen_table([report_dir])
-            logger.info(f'** Report table: \n {report_table} \n')
-        except Exception:
-            logger.error('Failed to generate report table.')
-
-    def gen_report(self, subset_score_map: dict, report_name: str = None) -> dict:
-        total_num: int = sum([num for _, num in subset_score_map.values()])
-        weighted_avg_acc: float = sum([score * num for score, num in subset_score_map.values()]) / total_num
-        weighted_avg_acc = normalize_score(score=weighted_avg_acc)
-        cate_avg_list = [{
-            'name': subset_name,
-            'score': normalize_score(score=score)
-        } for subset_name, (score, _) in subset_score_map.items()]
-
-        category_d = dict(name='DEFAULT', score=weighted_avg_acc, subset=cate_avg_list)
-
-        res_map = dict(
-            name=report_name or 'HumanEval',
-            metric='pass@1',
-            score=weighted_avg_acc,
-            category=[category_d],
-            total_num=total_num)
-
-        return res_map
-
     @classmethod
     def _postprocess(cls, text: str) -> str:
         if '```' in text:
@@ -182,19 +94,6 @@ class HumanevalAdapter(DataAdapter):
                 text = '\n'.join(['    ' + line for line in text.split('\n')])
         return text
 
-    def compute_metric(self, review_res_list: list) -> float:
-        """
-        Compute evaluation result by specific metric.
-
-        Args:
-            review_res_list: review score list, e.g. [0, 1, 1, 0, ...]
-
-        Returns:
-            The metric score.
-        """
-        items = [(score, 1.0) for score in review_res_list]
-        return weighted_mean(items)
-
     def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = 'checkpoint') -> str:
         return self._postprocess(result)
 

evalscope/benchmarks/mmlu/__init__.py

@@ -1,6 +1 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
-
-from evalscope.benchmarks.mmlu.mmlu_adapter import DATASET_ID, SUBJECT_MAPPING, SUBSET_LIST
-from evalscope.benchmarks.mmlu.mmlu_adapter import MMLUAdapter
-from evalscope.benchmarks.mmlu.mmlu_adapter import MMLUAdapter as DataAdapterClass
-from evalscope.models.model_adapter import MultiChoiceModelAdapter as ModelAdapterClass  # noqa

evalscope/benchmarks/mmlu/mmlu_adapter.py

@@ -2,8 +2,10 @@
 import csv
 import os
 
-from evalscope.benchmarks.data_adapter import DataAdapter
-from evalscope.metrics.metrics import exact_match, weighted_mean
+from evalscope.benchmarks import Benchmark, DataAdapter
+from evalscope.constants import EvalType
+from evalscope.metrics import WeightedAverageAccuracy, exact_match
+from evalscope.models import MultiChoiceModelAdapter
 from evalscope.utils import ResponseParser, normalize_score
 from evalscope.utils.logger import get_logger
 
@@ -134,40 +136,29 @@ SUBJECT_MAPPING = {
 }
 
 
+@Benchmark.register(
+    name='mmlu',
+    dataset_id='modelscope/mmlu',
+    model_adapter=MultiChoiceModelAdapter,
+    subset_list=SUBSET_LIST,
+    metric_list=[WeightedAverageAccuracy],
+    few_shot_num=5,
+    train_split='train',
+    eval_split='test',
+    prompt_template='',
+)
 class MMLUAdapter(DataAdapter):
 
     choices = ['A', 'B', 'C', 'D']
 
-    def __init__(self,
-                 subset_list: list = None,
-                 metric_list: list = None,
-                 few_shot_num: int = None,
-                 train_split: str = 'train',
-                 eval_split: str = 'test',
-                 **kwargs):
-
-        if subset_list is None:
-            subset_list = SUBSET_LIST
-
-        if metric_list is None:
-            metric_list = [{'name': 'WeightedAverageAccuracy', 'object': weighted_mean}]
-
-        if few_shot_num is None:
-            # Use 5-shot by default
-            logger.info(f'Set 5-shot examples by system for MMLU.')
-            few_shot_num = 5
+    def __init__(self, **kwargs):
 
+        few_shot_num = kwargs.get('few_shot_num', 5)
         if few_shot_num > 5:
             logger.warning(f'few_shot_num <= 5 for MMLU, but got {few_shot_num}. Use 5-shot by default.')
-            few_shot_num = 5
+            kwargs['few_shot_num'] = 5
 
-        super().__init__(
-            subset_list=subset_list,
-            metric_list=metric_list,
-            few_shot_num=few_shot_num,
-            train_split=train_split,
-            eval_split=eval_split,
-            **kwargs)
+        super().__init__(**kwargs)
 
     def load_from_disk(self, dataset_name_or_path, subset_list, work_dir, **kwargs) -> dict:
         data_dict = {}
@@ -225,7 +216,7 @@ class MMLUAdapter(DataAdapter):
             'target': 'A'}
 
         Returns:
-            {'data': [(context, continuation), ...]}
+            {'data': [full_prompt], 'multi_choices': self.choices}
 
         """
         prompt = 'The following are multiple choice questions (with answers) about {}.\n\n'.format(
@@ -244,7 +235,7 @@ class MMLUAdapter(DataAdapter):
         # Get the gold choice
         return input_d.get('target', '')
 
-    def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = 'checkpoint') -> str:
+    def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = EvalType.CHECKPOINT) -> str:
         """
         Parse the model output to get the answer. Could be the best choice index.
 
@@ -256,31 +247,18 @@ class MMLUAdapter(DataAdapter):
         Returns:
             The parsed answer. Depending on the dataset. Usually a string for chat.
         """
-        if eval_type == 'checkpoint':
+        if eval_type == EvalType.CHECKPOINT:
             return result
-        elif eval_type == 'service':
-            return ResponseParser.parse_first_option_with_choices(result, self.choices)  # TODO: to be checked !
-        elif eval_type == 'custom':
-            return ResponseParser.parse_first_option_with_choices(result, self.choices)  # TODO: to be checked !
+        elif eval_type == EvalType.SERVICE:
+            return ResponseParser.parse_first_option_with_choices(result, self.choices)
+        elif eval_type == EvalType.CUSTOM:
+            return ResponseParser.parse_first_option_with_choices(result, self.choices)
         else:
             raise ValueError(f'Invalid eval_type: {eval_type}')
 
     def match(self, gold: str, pred: str) -> float:
         return exact_match(gold=gold, pred=pred)
 
-    def compute_metric(self, review_res_list: list) -> float:
-        """
-        Compute evaluation result by specific metric.
-
-        Args:
-            review_res_list: review score list, e.g. [0, 1, 1, 0, ...]
-
-        Returns:
-            The metric score.
-        """
-        items = [(score, 1.0) for score in review_res_list]
-        return weighted_mean(items)
-
     def gen_report(self, subset_score_map: dict, report_name: str = None) -> dict:
         """
         Generate report for the evaluation.

evalscope/benchmarks/mmlu_pro/mmlu_pro_adapter.py

@@ -0,0 +1,110 @@
+from collections import defaultdict
+from typing import Any, Dict
+
+from evalscope.benchmarks import Benchmark, DataAdapter
+from evalscope.constants import AnswerKeys, EvalType
+from evalscope.metrics import WeightedAverageAccuracy, exact_match
+from evalscope.models import ChatGenerationModelAdapter
+from evalscope.utils.utils import ResponseParser
+
+
+@Benchmark.register(
+    name='mmlu_pro',
+    dataset_id='modelscope/mmlu-pro',
+    model_adapter=ChatGenerationModelAdapter,
+    subset_list=['default'],
+    metric_list=[WeightedAverageAccuracy],
+    few_shot_num=5,
+    train_split='validation',
+    eval_split='test',
+    prompt_template=
+    'You are an knowledge expert, you are supposed to answer the multi-choice question to derive your final answer as `The answer is ...`.',  # noqa: E501
+)
+class MMLUProAdapter(DataAdapter):
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+        self.choices = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J']
+        self.categories = [
+            'computer science', 'math', 'chemistry', 'engineering', 'law', 'biology', 'health', 'physics', 'business',
+            'philosophy', 'economics', 'other', 'psychology', 'history'
+        ]
+
+    def gen_prompts(self, data_dict: dict, **kwargs) -> Dict[str, list]:
+        """
+        Generate model prompt from raw input, unify the prompt format for MMLU-Pro benchmark.
+        Return a dict with category as key and list of prompts as value.
+        """
+
+        data_dict = data_dict[self.subset_list[0]]  # Only one subset for MMLU-Pro
+        fewshot_prompts = self.get_fewshot_examples(data_dict)
+
+        #  Use the category as key to group the prompts
+        res_dict = defaultdict(list)
+        # generate prompts for each test sample
+        for entry in data_dict[self.eval_split]:
+            prefix = fewshot_prompts[entry['category']]
+            query = prefix + 'Q: ' + entry['question'] + '\n' + \
+                self.__form_options(entry['options']) + '\n'
+
+            prompt_d = {'data': [query], 'system_prompt': self.prompt_template, AnswerKeys.RAW_INPUT: entry}
+
+            res_dict[entry['category']].append(prompt_d)
+        return res_dict
+
+    def get_fewshot_examples(self, data_dict: dict):
+        # load 5-shot prompts for each category
+        prompts = {c: '' for c in self.categories}
+        for d in data_dict[self.train_split]:
+            prompts[d['category']] += 'Q:' + ' ' + d['question'] + '\n' + \
+                self.__form_options(d['options']) + '\n' + \
+                d['cot_content'] + '\n\n'
+        return prompts
+
+    def __form_options(self, options: list):
+        option_str = 'Options are:\n'
+        for opt, choice in zip(options, self.choices):
+            option_str += f'({choice}): {opt}' + '\n'
+        return option_str
+
+    def get_gold_answer(self, input_d: dict) -> str:
+        """
+        Parse the raw input labels (gold).
+
+        Args:
+            input_d: input raw data. Depending on the dataset.
+
+        Returns:
+            The parsed input. e.g. gold answer ... Depending on the dataset.
+        """
+        return input_d['answer']
+
+    def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = EvalType.CHECKPOINT) -> str:
+        """
+        Parse the predicted result and extract proper answer.
+
+        Args:
+            result: Predicted answer from the model. Usually a string for chat.
+            raw_input_d: The raw input. Depending on the dataset.
+            eval_type: 'checkpoint' or 'service' or `custom`, default: 'checkpoint'
+
+        Returns:
+            The parsed answer. Depending on the dataset. Usually a string for chat.
+        """
+        return ResponseParser.parse_first_option(result)
+
+    def match(self, gold: str, pred: str) -> float:
+        """
+        Match the gold answer and the predicted answer.
+
+        Args:
+            gold (Any): The golden answer. Usually a string for chat/multiple-choice-questions.
+                        e.g. 'A', extracted from get_gold_answer method.
+            pred (Any): The predicted answer. Usually a string for chat/multiple-choice-questions.
+                        e.g. 'B', extracted from parse_pred_result method.
+
+        Returns:
+            The match result. Usually a score (float) for chat/multiple-choice-questions.
+        """
+        return exact_match(gold=gold, pred=pred)

evalscope/benchmarks/race/__init__.py

@@ -1,6 +1 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
-
-from evalscope.benchmarks.race.race_adapter import DATASET_ID, SUBJECT_MAPPING, SUBSET_LIST
-from evalscope.benchmarks.race.race_adapter import RACEAdapter
-from evalscope.benchmarks.race.race_adapter import RACEAdapter as DataAdapterClass
-from evalscope.models.model_adapter import MultiChoiceModelAdapter as ModelAdapterClass  # noqa

evalscope/benchmarks/race/race_adapter.py

@@ -1,11 +1,12 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 
-import json
 import os
 
-from evalscope.benchmarks.data_adapter import DataAdapter
-from evalscope.metrics.metrics import exact_match, weighted_mean
-from evalscope.utils import normalize_score
+from evalscope.benchmarks import Benchmark, DataAdapter
+from evalscope.constants import EvalType
+from evalscope.metrics import WeightedAverageAccuracy, exact_match
+from evalscope.models import MultiChoiceModelAdapter
+from evalscope.utils import ResponseParser, normalize_score
 from evalscope.utils.io_utils import jsonl_to_list
 from evalscope.utils.logger import get_logger
 
@@ -13,46 +14,30 @@ from evalscope.utils.logger import get_logger
 
 logger = get_logger()
 
-DATASET_ID = 'modelscope/race'
-
-SUBSET_LIST = ['high', 'middle']
-
 SUBJECT_MAPPING = {'high': 'High', 'middle': 'Middle'}
 
 
+@Benchmark.register(
+    name='race',
+    dataset_id='modelscope/race',
+    model_adapter=MultiChoiceModelAdapter,
+    subset_list=['high', 'middle'],
+    metric_list=[WeightedAverageAccuracy],
+    few_shot_num=3,
+    train_split='train',
+    eval_split='test',
+)
 class RACEAdapter(DataAdapter):
 
     choices = ['A', 'B', 'C', 'D']
 
-    def __init__(self,
-                 subset_list: list = None,
-                 metric_list: list = None,
-                 few_shot_num: int = None,
-                 train_split: str = 'train',
-                 eval_split: str = 'test',
-                 **kwargs):
-
-        if subset_list is None:
-            subset_list = SUBSET_LIST
-
-        if metric_list is None:
-            metric_list = [{'name': 'WeightedAverageAccuracy', 'object': weighted_mean}]
-
-        if few_shot_num is None:
-            logger.info(f'Set 3-shot examples by system for RACE.')
-            few_shot_num = 3
-
+    def __init__(self, **kwargs):
+        few_shot_num = kwargs.get('few_shot_num', 3)
         if few_shot_num > 3:
             logger.warning(f'few_shot_num <= 3 for RACE, but got {few_shot_num}. Use 3-shot by default.')
-            few_shot_num = 3
+            kwargs['few_shot_num'] = 3
 
-        super().__init__(
-            subset_list=subset_list,
-            metric_list=metric_list,
-            few_shot_num=few_shot_num,
-            train_split=train_split,
-            eval_split=eval_split,
-            **kwargs)
+        super().__init__(**kwargs)
 
     def load_from_disk(self, dataset_name_or_path, subset_list, work_dir, **kwargs) -> dict:
         data_dict = {}
@@ -105,7 +90,7 @@ class RACEAdapter(DataAdapter):
         # Get the gold choice
         return input_d.get('answer', '')
 
-    def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = 'checkpoint') -> str:
+    def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = EvalType.CHECKPOINT) -> str:
         """
         Parse the model output to get the answer. Could be the best choice index.
 
@@ -117,31 +102,18 @@ class RACEAdapter(DataAdapter):
         Returns:
             The parsed answer. Depending on the dataset. Usually a string for chat.
         """
-        if eval_type == 'checkpoint':
-            return result
-        elif eval_type == 'service':  # TODO: to be implemented
-            return result
-        elif eval_type == 'custom':  # TODO: to be implemented
+        if eval_type == EvalType.CHECKPOINT:
             return result
+        elif eval_type == EvalType.SERVICE:
+            return ResponseParser.parse_first_option_with_choices(result, self.choices)  # TODO: to be checked !
+        elif eval_type == EvalType.CUSTOM:
+            return ResponseParser.parse_first_option_with_choices(result, self.choices)  # TODO: to be checked !
         else:
             raise ValueError(f'Unknown eval_type: {eval_type}')
 
     def match(self, gold: str, pred: str) -> float:
         return exact_match(gold=gold, pred=pred)
 
-    def compute_metric(self, review_res_list: list) -> float:
-        """
-        Compute evaluation result by specific metric.
-
-        Args:
-            review_res_list: review score list, e.g. [0, 1, 1, 0, ...]
-
-        Returns:
-            The metric score.
-        """
-        items = [(score, 1.0) for score in review_res_list]
-        return weighted_mean(items)
-
     def gen_report(self, subset_score_map: dict, report_name: str = None) -> dict:
         """
         Generate report for the evaluation.

evalscope/benchmarks/trivia_qa/__init__.py

@@ -1,6 +1 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
-
-from evalscope.benchmarks.trivia_qa.trivia_qa_adapter import DATASET_ID, SUBSET_LIST
-from evalscope.benchmarks.trivia_qa.trivia_qa_adapter import TriviaQaAdapter
-from evalscope.benchmarks.trivia_qa.trivia_qa_adapter import TriviaQaAdapter as DataAdapterClass
-from evalscope.models.model_adapter import ChatGenerationModelAdapter as ModelAdapterClass  # noqa