evalscope 0.8.1__py3-none-any.whl → 0.9.0__py3-none-any.whl

evalscope/benchmarks/cmmlu/__init__.py

@@ -1,6 +1 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
-
-from evalscope.benchmarks.cmmlu.cmmlu_adapter import DATASET_ID, SUBJECT_MAPPING, SUBSET_LIST
-from evalscope.benchmarks.cmmlu.cmmlu_adapter import CMMLUAdapter
-from evalscope.benchmarks.cmmlu.cmmlu_adapter import CMMLUAdapter as DataAdapterClass
-from evalscope.models.model_adapter import MultiChoiceModelAdapter as ModelAdapterClass  # noqa

evalscope/benchmarks/cmmlu/cmmlu_adapter.py

@@ -3,8 +3,10 @@
 import csv
 import os
 
-from evalscope.benchmarks.data_adapter import DataAdapter
-from evalscope.metrics.metrics import exact_match, weighted_mean
+from evalscope.benchmarks import Benchmark, DataAdapter
+from evalscope.constants import EvalType
+from evalscope.metrics import WeightedAverageAccuracy, exact_match
+from evalscope.models import MultiChoiceModelAdapter
 from evalscope.utils import ResponseParser, normalize_score
 from evalscope.utils.logger import get_logger
 
@@ -12,8 +14,6 @@ from evalscope.utils.logger import get_logger
 
 logger = get_logger()
 
-DATASET_ID = 'modelscope/cmmlu'
-
 SUBSET_LIST = [
     'agronomy', 'anatomy', 'ancient_chinese', 'arts', 'astronomy', 'business_ethics', 'chinese_civil_service_exam',
     'chinese_driving_rule', 'chinese_food_culture', 'chinese_foreign_policy', 'chinese_history', 'chinese_literature',
@@ -101,31 +101,23 @@ SUBJECT_MAPPING = {
 }
 
 
+@Benchmark.register(
+    name='cmmlu',
+    dataset_id='modelscope/cmmlu',
+    model_adapter=MultiChoiceModelAdapter,
+    subset_list=SUBSET_LIST,
+    metric_list=[WeightedAverageAccuracy],
+    few_shot_num=5,
+    train_split='dev',
+    eval_split='test',
+)
 class CMMLUAdapter(DataAdapter):
 
     choices = ['A', 'B', 'C', 'D']
 
-    def __init__(self,
-                 subset_list: list = None,
-                 metric_list: list = None,
-                 few_shot_num: int = 5,
-                 train_split: str = 'dev',
-                 eval_split: str = 'test',
-                 **kwargs):
-
-        if subset_list is None:
-            subset_list = SUBSET_LIST
-
-        if metric_list is None:
-            metric_list = [{'name': 'WeightedAverageAccuracy', 'object': weighted_mean}]
+    def __init__(self, **kwargs):
 
-        super().__init__(
-            subset_list=subset_list,
-            metric_list=metric_list,
-            few_shot_num=few_shot_num,
-            train_split=train_split,
-            eval_split=eval_split,
-            **kwargs)
+        super().__init__(**kwargs)
 
     def load_from_disk(self, dataset_name_or_path, subset_list, work_dir, **kwargs) -> dict:
         data_dict = {}
@@ -187,7 +179,7 @@ class CMMLUAdapter(DataAdapter):
         # Get the gold choice
         return input_d.get('Answer', '')
 
-    def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = 'checkpoint') -> str:
+    def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = EvalType.CHECKPOINT) -> str:
         """
         Parse the model output to get the answer. Could be the best choice index.
 
@@ -199,11 +191,11 @@ class CMMLUAdapter(DataAdapter):
         Returns:
             The parsed answer. Depending on the dataset. Usually a string for chat.
         """
-        if eval_type == 'checkpoint':
+        if eval_type == EvalType.CHECKPOINT:
             return result
-        elif eval_type == 'service':
+        elif eval_type == EvalType.SERVICE:
             return ResponseParser.parse_first_option_with_choices(result, self.choices)  # TODO: to be checked !
-        elif eval_type == 'custom':
+        elif eval_type == EvalType.CUSTOM:
             return ResponseParser.parse_first_option_with_choices(result, self.choices)  # TODO: to be checked !
         else:
             raise ValueError(f'Invalid eval_type: {eval_type}')
@@ -211,19 +203,6 @@ class CMMLUAdapter(DataAdapter):
     def match(self, gold: str, pred: str) -> float:
         return exact_match(gold=gold, pred=pred)
 
-    def compute_metric(self, review_res_list: list) -> float:
-        """
-        Compute evaluation result by specific metric.
-
-        Args:
-            review_res_list: review score list, e.g. [0, 1, 1, 0, ...]
-
-        Returns:
-            The metric score.
-        """
-        items = [(score, 1.0) for score in review_res_list]
-        return weighted_mean(items)
-
     def gen_report(self, subset_score_map: dict, report_name: str = None) -> dict:
         """
         Generate report for the evaluation.

evalscope/benchmarks/competition_math/__init__.py

@@ -1,6 +1 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
-
-from evalscope.benchmarks.competition_math.competition_math_adapter import DATASET_ID, SUBSET_LIST
-from evalscope.benchmarks.competition_math.competition_math_adapter import CompetitionMathAdapter
-from evalscope.benchmarks.competition_math.competition_math_adapter import CompetitionMathAdapter as DataAdapterClass
-from evalscope.models.model_adapter import ChatGenerationModelAdapter as ModelAdapterClass  # noqa

evalscope/benchmarks/competition_math/competition_math_adapter.py

@@ -4,53 +4,40 @@ import glob
 import json
 import os
 
-from evalscope.benchmarks import DataAdapter
-from evalscope.metrics.metrics import weighted_mean
-from evalscope.utils import normalize_score
+from evalscope.benchmarks import Benchmark, DataAdapter
+from evalscope.metrics import WeightedAverageAccuracy
+from evalscope.metrics.math_accuracy import is_equiv, last_boxed_only_string, remove_boxed
+from evalscope.models import ChatGenerationModelAdapter
 from evalscope.utils.logger import get_logger
 
 # flake8: noqa
 
 logger = get_logger()
 
-DATASET_ID = 'modelscope/competition_math'
-SUBSET_LIST = ['default']
-
 
+@Benchmark.register(
+    name='competition_math',
+    dataset_id='modelscope/competition_math',
+    model_adapter=ChatGenerationModelAdapter,
+    subset_list=['default'],
+    metric_list=[WeightedAverageAccuracy],
+    few_shot_num=4,
+    train_split='train',
+    eval_split='test',
+    prompt_template='',
+)
 class CompetitionMathAdapter(DataAdapter):
-    """ TODO: To be tested for all models. """
-
-    def __init__(self,
-                 subset_list: list = None,
-                 metric_list: list = None,
-                 few_shot_num: int = None,
-                 train_split: str = 'train',
-                 eval_split: str = 'test',
-                 **kwargs):
-
-        if subset_list is None:
-            subset_list = SUBSET_LIST
+    """ To be tested for all models. """
 
-        if metric_list is None:
-            metric_list = [{'name': 'WeightedAverageAccuracy', 'object': weighted_mean}]
-
-        if few_shot_num is None:
-            # Use 4-shot by default
-            logger.info(f'Set 4-shot examples by system for MATH.')
-            few_shot_num = 4
+    def __init__(self, **kwargs):
 
+        few_shot_num = kwargs.get('few_shot_num', 4)
         if few_shot_num != 4 and few_shot_num != 0:
             logger.error(f'The MATH benchmark ONLY supports 4-shot by system or 0-shot settings, '
-                         f'but got {self.few_shot_num}. Use 4-shot by default.')
-            few_shot_num = 4
+                         f'but got {few_shot_num}. Use 4-shot by default.')
+            kwargs['few_shot_num'] = 4
 
-        super().__init__(
-            subset_list=subset_list,
-            metric_list=metric_list,
-            few_shot_num=few_shot_num,
-            train_split=train_split,
-            eval_split=eval_split,
-            **kwargs)
+        super().__init__(**kwargs)
 
     def load_from_disk(self, dataset_name_or_path, subset_list, work_dir, **kwargs) -> dict:
         data_dict: dict = {}
@@ -90,11 +77,11 @@ class CompetitionMathAdapter(DataAdapter):
         use_fewshot = self.few_shot_num > 0
         full_prompt = self._generate_prompt(input_d, use_fewshot=use_fewshot)
 
-        return {'data': [full_prompt]}
+        return {'data': [full_prompt], 'system_prompt': 'Put the final answer in \\boxed{}.'}
 
     def get_gold_answer(self, input_d: dict) -> str:
         # Extract the gold answer from the input dict.
-        return self._preprocess_input(input_d['solution'])
+        return remove_boxed(last_boxed_only_string(input_d['solution']))
 
     def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = 'checkpoint') -> str:
         """
@@ -108,77 +95,20 @@ class CompetitionMathAdapter(DataAdapter):
         Returns:
             The parsed answer. Depending on the dataset. Usually a string for chat.
         """
-        # TODO: check answer extraction
         # Note: Use same extraction method for both of checkpoint/service/custom
-        return self._math_postprocess(result)
+        try:
+            result = remove_boxed(last_boxed_only_string(result))
+        except Exception:
+            return None
+        return result
 
     def match(self, gold: str, pred: str) -> float:
         res = 0
-        if self._is_equiv(pred, gold):
+        if is_equiv(pred, gold):
             res = 1
 
         return res
 
-    def compute_metric(self, review_res_list: list) -> float:
-        """
-        Compute evaluation result by specific metric.
-
-        Args:
-            review_res_list: review score list, e.g. [0, 1, 1, 0, ...]
-
-        Returns:
-            The metric score.
-        """
-        items = [(score, 1.0) for score in review_res_list]
-        return weighted_mean(items)
-
-    def gen_report(self, subset_score_map: dict, report_name: str = None) -> dict:
-        """
-        Generate the report for the model output.
-
-        Args:
-            subset_score_map: The subset-score mapping. e.g. {subset_name: (score, num), ...}
-            report_name: The user-defined report name.
-
-        Returns: A dict of metric calculation results. The format is like:
-        {
-            "name":"CompetitionMath",
-            "metric":"WeightedAverageAccuracy",
-            "score":0.5632,
-            "category":[
-                {
-                    "name":"DEFAULT",
-                    "score":0.5632,
-                    "subset":[
-                        {
-                            "name":"main",
-                            "score":0.5632
-                        },
-                    ]
-                }
-            ],
-            "total_num":100
-        }
-        """
-        total_num: int = sum([num for _, num in subset_score_map.values()])
-        weighted_avg_acc: float = sum([score * num for score, num in subset_score_map.values()]) / total_num
-        weighted_avg_acc = normalize_score(score=weighted_avg_acc)
-        cate_avg_list = [{
-            'name': subset_name,
-            'score': normalize_score(score=score)
-        } for subset_name, (score, _) in subset_score_map.items()]
-
-        category_d = dict(name='DEFAULT', score=weighted_avg_acc, subset=cate_avg_list)
-
-        res_map = dict(
-            name=report_name or 'competition_math',
-            metric=self.metric_list[0]['name'],
-            score=weighted_avg_acc,
-            category=[category_d],
-            total_num=total_num)
-
-        return res_map
-
     @classmethod
     def _generate_prompt(cls, input_d: dict, use_fewshot: bool = True) -> str:
         problem: str = input_d['problem']
@@ -194,275 +124,3 @@ class CompetitionMathAdapter(DataAdapter):
         else:
             context = 'Problem:\n' + problem + '\nSolution:\n'
         return context
-
-    @classmethod
-    def _preprocess_input(cls, input: str) -> str:
-        """
-        Preprocess the input data, remove the boxed solution.
-
-        Args:
-            input_d: The raw input. A single data format of the Competition Math.
-
-        Returns:
-            The preprocessed input.
-        """
-        return cls._remove_boxed(cls._last_boxed_only_string(input))
-
-    @classmethod
-    def _remove_boxed(cls, s):
-        if s is None:
-            return s
-
-        if '\\boxed ' in s:
-            left = '\\boxed '
-            assert s[:len(left)] == left
-            return s[len(left):]
-
-        left = '\\boxed{'
-
-        assert s[:len(left)] == left
-        assert s[-1] == '}'
-
-        return s[len(left):-1]
-
-    @classmethod
-    def _last_boxed_only_string(cls, string):
-
-        idx = string.rfind('\\boxed')
-        if '\\boxed ' in string:
-            return '\\boxed ' + string.split('\\boxed ')[-1].split('$')[0]
-        if idx < 0:
-            idx = string.rfind('\\fbox')
-            if idx < 0:
-                return None
-
-        i = idx
-        right_brace_idx = None
-        num_left_braces_open = 0
-        while i < len(string):
-            if string[i] == '{':
-                num_left_braces_open += 1
-            if string[i] == '}':
-                num_left_braces_open -= 1
-                if num_left_braces_open == 0:
-                    right_brace_idx = i
-                    break
-            i += 1
-
-        if right_brace_idx is None:
-            retval = None
-        else:
-            retval = string[idx:right_brace_idx + 1]
-
-        return retval
-
-    @classmethod
-    def _is_equiv(cls, str1, str2, verbose=False):
-        if str1 is None and str2 is None:
-            logger.warning('WARNING: Both None')
-            return True
-        if str1 is None or str2 is None:
-            return False
-
-        try:
-            ss1 = cls.strip_string(str1)
-            ss2 = cls.strip_string(str2)
-            if verbose:
-                logger.info(f'ss1: {ss1}, ss2: {ss2}')
-            return ss1 == ss2
-        except Exception:
-            return str1 == str2
-
-    @classmethod
-    def strip_string(cls, string):
-        # linebreaks
-        string = string.replace('\n', '')
-
-        # remove inverse spaces
-        string = string.replace('\\!', '')
-
-        # replace \\ with \
-        string = string.replace('\\\\', '\\')
-
-        # replace tfrac and dfrac with frac
-        string = string.replace('tfrac', 'frac')
-        string = string.replace('dfrac', 'frac')
-
-        # remove \left and \right
-        string = string.replace('\\left', '')
-        string = string.replace('\\right', '')
-
-        # Remove circ (degrees)
-        string = string.replace('^{\\circ}', '')
-        string = string.replace('^\\circ', '')
-
-        # remove dollar signs
-        string = string.replace('\\$', '')
-
-        # remove units (on the right)
-        string = cls.remove_right_units(string)
-
-        # remove percentage
-        string = string.replace('\\%', '')
-        string = string.replace('\%', '')  # noqa: W605
-
-        # " 0." equivalent to " ." and "{0." equivalent to "{." Alternatively, add "0" if "." is the start of the string
-        string = string.replace(' .', ' 0.')
-        string = string.replace('{.', '{0.')
-        # if empty, return empty string
-        if len(string) == 0:
-            return string
-        if string[0] == '.':
-            string = '0' + string
-
-        # to consider: get rid of e.g. "k = " or "q = " at beginning
-        if len(string.split('=')) == 2:
-            if len(string.split('=')[0]) <= 2:
-                string = string.split('=')[1]
-
-        # fix sqrt3 --> sqrt{3}
-        string = cls.fix_sqrt(string)
-
-        # remove spaces
-        string = string.replace(' ', '')
-
-        # \frac1b or \frac12 --> \frac{1}{b} and \frac{1}{2}, etc. Even works with \frac1{72} (but not \frac{72}1). Also does a/b --> \\frac{a}{b}
-        string = cls.fix_fracs(string)
-
-        # manually change 0.5 --> \frac{1}{2}
-        if string == '0.5':
-            string = '\\frac{1}{2}'
-
-        # NOTE: X/Y changed to \frac{X}{Y} in dataset, but in simple cases fix in case the model output is X/Y
-        string = cls.fix_a_slash_b(string)
-
-        return string
-
-    @classmethod
-    def remove_right_units(cls, string):
-        # "\\text{ " only ever occurs (at least in the val set) when describing units
-        if '\\text{ ' in string:
-            splits = string.split('\\text{ ')
-            assert len(splits) == 2
-            return splits[0]
-        else:
-            return string
-
-    @classmethod
-    def fix_fracs(cls, string):
-        substrs = string.split('\\frac')
-        new_str = substrs[0]
-        if len(substrs) > 1:
-            substrs = substrs[1:]
-            for substr in substrs:
-                new_str += '\\frac'
-                if substr[0] == '{':
-                    new_str += substr
-                else:
-                    try:
-                        assert len(substr) >= 2
-                    except AssertionError:
-                        return string
-                    a = substr[0]
-                    b = substr[1]
-                    if b != '{':
-                        if len(substr) > 2:
-                            post_substr = substr[2:]
-                            new_str += '{' + a + '}{' + b + '}' + post_substr
-                        else:
-                            new_str += '{' + a + '}{' + b + '}'
-                    else:
-                        if len(substr) > 2:
-                            post_substr = substr[2:]
-                            new_str += '{' + a + '}' + b + post_substr
-                        else:
-                            new_str += '{' + a + '}' + b
-        string = new_str
-        return string
-
-    @classmethod
-    def fix_sqrt(cls, string):
-        if '\\sqrt' not in string:
-            return string
-        splits = string.split('\\sqrt')
-        new_string = splits[0]
-        for split in splits[1:]:
-            if split[0] != '{':
-                a = split[0]
-                new_substr = '\\sqrt{' + a + '}' + split[1:]
-            else:
-                new_substr = '\\sqrt' + split
-            new_string += new_substr
-        return new_string
-
-    @classmethod
-    def fix_a_slash_b(cls, string):
-        if len(string.split('/')) != 2:
-            return string
-        a = string.split('/')[0]
-        b = string.split('/')[1]
-        try:
-            a = int(a)
-            b = int(b)
-            assert string == '{}/{}'.format(a, b)
-            new_string = '\\frac{' + str(a) + '}{' + str(b) + '}'
-            return new_string
-        except AssertionError:
-            return string
-
-    @classmethod
-    def _math_postprocess(cls, text: str) -> str:
-        SUBSTITUTIONS = [('an ', ''), ('a ', ''), ('.$', '$'), ('\\$', ''), (r'\ ', ''), (' ', ''), ('mbox', 'text'),
-                         (',\\text{and}', ','), ('\\text{and}', ','), ('\\text{m}', '\\text{}'), ('\\le', '<')]
-        REMOVED_EXPRESSIONS = [
-            'square', 'ways', 'integers', 'dollars', 'mph', 'inches', 'ft', 'hours', 'km', 'units', '\\ldots', 'sue',
-            'points', 'feet', 'minutes', 'digits', 'cents', 'degrees', 'cm', 'gm', 'pounds', 'meters', 'meals', 'edges',
-            'students', 'childrentickets', 'multiples', '\\text{s}', '\\text{.}', '\\text{\ns}', '\\text{}^2',
-            '\\text{}^3', '\\text{\n}', '\\text{}', r'\mathrm{th}', r'^\circ', r'^{\circ}', r'\;', r',\!', '{,}', '"',
-            '\\dots', '\n', '\r', '\f'
-        ]
-        import re
-
-        def normalize_final_answer(final_answer: str) -> str:
-            """Normalize a final answer to a quantitative reasoning question."""
-            # final_answer = final_answer.split('=')[-1]
-            for before, after in SUBSTITUTIONS:
-                final_answer = final_answer.replace(before, after)
-            for expr in REMOVED_EXPRESSIONS:
-                final_answer = final_answer.replace(expr, '')
-
-            # Extract answer that is in LaTeX math, is bold,
-            # is surrounded by a box, etc.
-            final_answer = re.sub(r'(\\text\{)(.*?)(\})', '\\2', final_answer)
-            final_answer = re.sub(r'(\\textbf\{)(.*?)(\})', '\\2', final_answer)
-            final_answer = re.sub(r'(\\overline\{)(.*?)(\})', '\\2', final_answer)
-            final_answer = re.sub(r'(\\boxed\{)(.*)(\})', '\\2', final_answer)
-            assert '\n' not in final_answer
-            assert '\r' not in final_answer
-            assert '\f' not in final_answer
-            if len(re.findall(r'finalansweris(.*)', final_answer)) > 0:
-                final_answer = re.findall(r'finalansweris(.*)', final_answer)[-1]
-
-            if len(re.findall(r'oxed\{(.*?)\}', final_answer)) > 0:
-                final_answer = re.findall(r'oxed\{(.*?)\}', final_answer)[-1]
-
-            if len(re.findall(r'\$(.*?)\$', final_answer)) > 0:
-                final_answer = re.findall(r'\$(.*?)\$', final_answer)[-1]
-            final_answer = final_answer.strip()
-            if 'rac' in final_answer and '\\frac' not in final_answer:
-                final_answer = final_answer.replace('rac', '\\frac')
-
-            final_answer = re.sub(r'(frac)([^{])(.)', 'frac{\\2}{\\3}', final_answer)
-            final_answer = re.sub(r'(sqrt)([^{])', 'sqrt{\\2}', final_answer)
-            final_answer = final_answer.replace('$', '')
-
-            # Normalize 100,000 -> 100000
-            if final_answer.replace(',', '').isdigit():
-                final_answer = final_answer.replace(',', '')
-
-            return final_answer
-
-        for maybe_ans in text.split('.'):
-            if 'final answer' in maybe_ans.lower():
-                return normalize_final_answer(maybe_ans)
-        return normalize_final_answer(text.split('.')[0])