evalscope 0.11.0__py3-none-any.whl → 0.12.1__py3-none-any.whl

evalscope/benchmarks/mmlu/mmlu_adapter.py

@@ -3,9 +3,8 @@ import csv
 import os
 
 from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.constants import EvalType
+from evalscope.constants import EvalType, OutputType
 from evalscope.metrics import exact_match
-from evalscope.models import MultiChoiceModelAdapter
 from evalscope.utils import ResponseParser
 from evalscope.utils.logger import get_logger
 
@@ -136,8 +135,10 @@ SUBJECT_MAPPING = {
 
 @Benchmark.register(
     name='mmlu',
+    pretty_name='MMLU',
     dataset_id='modelscope/mmlu',
-    model_adapter=MultiChoiceModelAdapter,
+    model_adapter=OutputType.MULTIPLE_CHOICE,
+    output_types=[OutputType.MULTIPLE_CHOICE, OutputType.GENERATION],
     subset_list=SUBSET_LIST,
     metric_list=['AverageAccuracy'],
     few_shot_num=5,
@@ -147,8 +148,6 @@ SUBJECT_MAPPING = {
 )
 class MMLUAdapter(DataAdapter):
 
-    choices = ['A', 'B', 'C', 'D']
-
     def __init__(self, **kwargs):
 
         few_shot_num = kwargs.get('few_shot_num', 5)
@@ -159,6 +158,7 @@ class MMLUAdapter(DataAdapter):
         super().__init__(**kwargs)
 
         self.category_map = {k: v[-1] for k, v in SUBJECT_MAPPING.items()}
+        self.choices = ['A', 'B', 'C', 'D']
 
     def load_from_disk(self, dataset_name_or_path, subset_list, work_dir, **kwargs) -> dict:
         data_dict = {}
@@ -227,7 +227,7 @@ class MMLUAdapter(DataAdapter):
 
         full_prompt = self.prompt_template.format(subset_name=self._format_subject(subset_name), query=query)
 
-        return {'data': [full_prompt], 'multi_choices': self.choices, 'system_prompt': self.system_prompt}
+        return self.gen_prompt_data(full_prompt)
 
     def get_gold_answer(self, input_d: dict) -> str:
         # Get the gold choice
@@ -245,26 +245,21 @@ class MMLUAdapter(DataAdapter):
         Returns:
             The parsed answer. Depending on the dataset. Usually a string for chat.
         """
-        if eval_type == EvalType.CHECKPOINT:
+        if self.model_adapter == OutputType.MULTIPLE_CHOICE:
             return result
-        elif eval_type == EvalType.SERVICE:
-            return ResponseParser.parse_first_option_with_choices(result, self.choices)
-        elif eval_type == EvalType.CUSTOM:
-            return ResponseParser.parse_first_option_with_choices(result, self.choices)
         else:
-            raise ValueError(f'Invalid eval_type: {eval_type}')
+            return ResponseParser.parse_first_option(result, self.choices)
 
     def match(self, gold: str, pred: str) -> float:
         return exact_match(gold=gold, pred=pred)
 
-    @classmethod
-    def _generate_prompt(cls, input_d: dict, include_answer=True) -> str:
+    def _generate_prompt(self, input_d: dict, include_answer=True) -> str:
 
         input_choices: list = [input_d['A'], input_d['B'], input_d['C'], input_d['D']]
 
         example: str = input_d['input']
-        for j in range(len(cls.choices)):
-            example += '\n{}. {}'.format(cls.choices[j], input_choices[j])
+        for j in range(len(self.choices)):
+            example += '\n{}. {}'.format(self.choices[j], input_choices[j])
 
         example += '\nAnswer:'
         if include_answer:

evalscope/benchmarks/mmlu_pro/mmlu_pro_adapter.py

@@ -2,9 +2,8 @@ from collections import defaultdict
 from typing import Any, Dict
 
 from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.constants import AnswerKeys, EvalType
+from evalscope.constants import EvalType, OutputType
 from evalscope.metrics import exact_match
-from evalscope.models import ChatGenerationModelAdapter
 from evalscope.utils.utils import ResponseParser
 
 SUBSET_LIST = [
@@ -15,8 +14,10 @@ SUBSET_LIST = [
 
 @Benchmark.register(
     name='mmlu_pro',
-    dataset_id='modelscope/mmlu-pro',
-    model_adapter=ChatGenerationModelAdapter,
+    pretty_name='MMLU-Pro',
+    dataset_id='modelscope/MMLU-Pro',
+    model_adapter=OutputType.GENERATION,
+    output_types=[OutputType.MULTIPLE_CHOICE, OutputType.GENERATION],
     subset_list=SUBSET_LIST,
     metric_list=['AverageAccuracy'],
     few_shot_num=5,
@@ -35,41 +36,25 @@ class MMLUProAdapter(DataAdapter):
     def load(self, **kwargs):
         # default load all data
         kwargs['subset_list'] = ['default']
-        return super().load(**kwargs)
+        data_dict = super().load(**kwargs)
+        return self.reformat_subset(data_dict, subset_key='category')
 
-    def gen_prompts(self, data_dict: dict, **kwargs) -> Dict[str, list]:
-        """
-        Generate model prompt from raw input, unify the prompt format for MMLU-Pro benchmark.
-        Return a dict with category as key and list of prompts as value.
-        """
-
-        data_dict = data_dict['default']  # Only one subset for MMLU-Pro
-        fewshot_prompts = self.get_fewshot_examples(data_dict)
-
-        #  Use the category as key to group the prompts
-        res_dict = defaultdict(list)
-        # generate prompts for each test sample
-        for entry in data_dict[self.eval_split]:
-            subset_name = entry['category']
-            if subset_name not in self.subset_list:
-                continue
-            prefix = fewshot_prompts[subset_name]
-            query = prefix + 'Q: ' + entry['question'] + '\n' + \
-                self.__form_options(entry['options']) + '\n'
-
-            full_prompt = self.prompt_template.format(subset_name=subset_name, query=query)
-            prompt_d = {'data': [full_prompt], 'system_prompt': self.system_prompt, AnswerKeys.RAW_INPUT: entry}
+    def gen_prompt(self, input_d: Dict, subset_name: str, few_shot_list: list, **kwargs) -> Any:
+        if self.few_shot_num > 0:
+            prefix = self.format_fewshot_examples(few_shot_list)
+        else:
+            prefix = ''
+        query = prefix + 'Q: ' + input_d['question'] + '\n' + \
+            self.__form_options(input_d['options']) + '\n'
 
-            res_dict[subset_name].append(prompt_d)
-        return res_dict
+        full_prompt = self.prompt_template.format(subset_name=subset_name, query=query)
+        return self.gen_prompt_data(full_prompt)
 
-    def get_fewshot_examples(self, data_dict: dict):
+    def format_fewshot_examples(self, few_shot_list):
         # load few-shot prompts for each category
-        prompts = {c: '' for c in self.subset_list}
-        for index, d in enumerate(data_dict[self.train_split]):
-            if index >= self.few_shot_num:
-                break
-            prompts[d['category']] += 'Q:' + ' ' + d['question'] + '\n' + \
+        prompts = ''
+        for index, d in enumerate(few_shot_list):
+            prompts += 'Q: ' + d['question'] + '\n' + \
                 self.__form_options(d['options']) + '\n' + \
                 d['cot_content'] + '\n\n'
         return prompts
@@ -104,7 +89,10 @@ class MMLUProAdapter(DataAdapter):
         Returns:
             The parsed answer. Depending on the dataset. Usually a string for chat.
         """
-        return ResponseParser.parse_first_option(result)
+        if self.model_adapter == OutputType.MULTIPLE_CHOICE:
+            return result
+        else:
+            return ResponseParser.parse_first_option(result)
 
     def match(self, gold: str, pred: str) -> float:
         """

evalscope/benchmarks/musr/musr_adapter.py

@@ -0,0 +1,71 @@
+import ast
+from typing import Any
+
+from evalscope.benchmarks import Benchmark, DataAdapter
+from evalscope.constants import EvalType, OutputType
+from evalscope.metrics import exact_match
+from evalscope.utils.utils import ResponseParser
+
+
+@Benchmark.register(
+    name='musr',
+    pretty_name='MuSR',
+    dataset_id='AI-ModelScope/MuSR',
+    model_adapter=OutputType.GENERATION,
+    output_types=[OutputType.MULTIPLE_CHOICE, OutputType.GENERATION],
+    subset_list=['murder_mysteries', 'object_placements', 'team_allocation'],
+    metric_list=['AverageAccuracy'],
+    few_shot_num=0,
+    train_split=None,
+    eval_split='test',
+    prompt_template=
+    '{narrative}\n\n{question}\n\n{choices}\nThink step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.',  # noqa: E501
+)
+class MuSRAdapter(DataAdapter):
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+        self.choices = ['A', 'B', 'C', 'D', 'E', 'F']
+
+    def load(self, **kwargs):
+        # default load all levels
+        kwargs['split_as_subset'] = True
+        data_dict = super().load(**kwargs)
+        return data_dict
+
+    def gen_prompt(self, input_d: dict, subset_name: str, few_shot_list: list, **kwargs) -> Any:
+
+        choices = self.format_choice(ast.literal_eval(input_d['choices']))
+
+        full_prompt = self.prompt_template.format(
+            narrative=input_d['narrative'], question=input_d['question'], choices=choices)
+
+        return self.gen_prompt_data(full_prompt)
+
+    def format_choice(self, options: list):
+        option_str = ''
+        for opt, choice in zip(options, self.choices):
+            option_str += f'({choice}): {opt}\n'
+        return option_str
+
+    def get_gold_answer(self, input_d: dict) -> str:
+        """
+        Parse the raw input labels (gold).
+        """
+        return self.choices[input_d['answer_index']]
+
+    def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = EvalType.CHECKPOINT) -> str:
+        """
+        Parse the predicted result and extract proper answer.
+        """
+        if self.model_adapter == OutputType.MULTIPLE_CHOICE:
+            return result
+        else:
+            return ResponseParser.parse_first_option(result)
+
+    def match(self, gold: str, pred: str) -> float:
+        """
+        Match the gold answer and the predicted answer.
+        """
+        return exact_match(gold=gold, pred=pred)

evalscope/benchmarks/process_bench/critique_template.txt

@@ -0,0 +1,13 @@
+The following is a math problem and a solution (split into paragraphs, enclosed with tags and indexed from 0):
+
+[Math Problem]
+
+{problem}
+
+[Solution]
+
+{tagged_response}
+
+Your task is to review and critique the solution paragraph by paragraph. Once you identify an error in a paragraph, return the index of the paragraph where the earliest error occurs. Otherwise, return the index of -1 (which typically denotes "not found").
+
+Please put your final answer (i.e., the index) in \boxed{{}}.

evalscope/benchmarks/process_bench/process_bench_adapter.py

@@ -0,0 +1,99 @@
+import os
+import re
+from typing import Any, List
+
+from evalscope.benchmarks import Benchmark, DataAdapter
+from evalscope.constants import AnswerKeys, EvalType
+from evalscope.metrics import Metric, mean, metric_registry, simple_f1_score
+
+cur_path = os.path.dirname(os.path.abspath(__file__))
+
+
+@Benchmark.register(
+    name='process_bench',
+    pretty_name='ProcessBench',
+    dataset_id='Qwen/ProcessBench',
+    subset_list=['gsm8k', 'math', 'olympiadbench', 'omnimath'],
+    metric_list=['error_acc', 'correct_acc', 'simple_f1_score'],
+    few_shot_num=0,
+    train_split=None,
+    eval_split='test',
+)
+class ProcessBenchAdapter(DataAdapter):
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+        self.prompt_template = open(os.path.join(cur_path, 'critique_template.txt'), encoding='utf-8').read()
+
+        # register metrics
+        metric_registry.register(Metric(name='error_acc', object=mean))
+        metric_registry.register(Metric(name='correct_acc', object=mean))
+        metric_registry.register(Metric(name='simple_f1_score', object=simple_f1_score))
+
+    def load(self, **kwargs):
+        # default load all levels
+        kwargs['split_as_subset'] = True
+        data_dict = super().load(**kwargs)
+        return data_dict
+
+    def gen_prompt(self, input_d: dict, subset_name: str, few_shot_list: list, **kwargs) -> Any:
+
+        problem = input_d['problem']
+        steps = input_d['steps']
+        tagged_response = ''
+        for sdx, step in enumerate(steps):
+            tagged_response += f'<paragraph_{sdx}>\n{step}\n</paragraph_{sdx}>\n\n'
+        tagged_response = tagged_response.strip()
+
+        full_prompt = self.prompt_template.format(problem=problem, tagged_response=tagged_response)
+
+        return self.gen_prompt_data(full_prompt)
+
+    def get_gold_answer(self, input_d: dict) -> str:
+        """
+        Parse the raw input labels (gold).
+        """
+        return int(input_d['label'])
+
+    def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = EvalType.CHECKPOINT) -> str:
+        """
+        Parse the predicted result and extract proper answer.
+        """
+        pred = ProcessBenchAdapter.extract_answer(result)
+        try:
+            pred = int(pred)
+        except Exception:
+            pred = None
+        return pred
+
+    def match(self, gold: int, pred: int) -> float:
+        """
+        Match the gold answer and the predicted answer.
+        """
+        return gold == pred
+
+    def compute_metric(self, review_res_list: list, **kwargs) -> List[dict]:
+        reviews_list = kwargs['reviews_list']
+        error_data = []
+        correct_data = []
+        for res, raw in zip(review_res_list, reviews_list):
+            if raw[AnswerKeys.RAW_INPUT]['label'] == -1:
+                correct_data.append(res)
+            else:
+                error_data.append(res)
+        data = {}
+        if len(correct_data) != 0:
+            data.update({'correct_acc': correct_data})
+        if len(error_data) != 0:
+            data.update({'error_acc': error_data})
+        data.update({'simple_f1_score': (correct_data, error_data)})
+        return super().compute_metric(data)
+
+    @staticmethod
+    def extract_answer(solution_text: str):
+        boxed_pattern = r'\\boxed\{([^}]*)\}'
+        matches = re.findall(boxed_pattern, solution_text)
+        if matches:
+            return matches[-1].strip()
+        return None

evalscope/benchmarks/race/race_adapter.py

@@ -3,9 +3,8 @@
 import os
 
 from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.constants import EvalType
+from evalscope.constants import EvalType, OutputType
 from evalscope.metrics import exact_match
-from evalscope.models import MultiChoiceModelAdapter
 from evalscope.utils import ResponseParser
 from evalscope.utils.io_utils import jsonl_to_list
 from evalscope.utils.logger import get_logger
@@ -17,8 +16,10 @@ logger = get_logger()
 
 @Benchmark.register(
     name='race',
+    pretty_name='RACE',
     dataset_id='modelscope/race',
-    model_adapter=MultiChoiceModelAdapter,
+    model_adapter=OutputType.MULTIPLE_CHOICE,
+    output_types=[OutputType.MULTIPLE_CHOICE, OutputType.GENERATION],
     subset_list=['high', 'middle'],
     metric_list=['AverageAccuracy'],
     few_shot_num=3,
@@ -27,8 +28,6 @@ logger = get_logger()
 )
 class RACEAdapter(DataAdapter):
 
-    choices = ['A', 'B', 'C', 'D']
-
     def __init__(self, **kwargs):
         few_shot_num = kwargs.get('few_shot_num', 3)
         if few_shot_num > 3:
@@ -37,6 +36,8 @@ class RACEAdapter(DataAdapter):
 
         super().__init__(**kwargs)
 
+        self.choices = ['A', 'B', 'C', 'D']
+
     def load_from_disk(self, dataset_name_or_path, subset_list, work_dir, **kwargs) -> dict:
         data_dict = {}
         for subset_name in subset_list:
@@ -82,7 +83,7 @@ class RACEAdapter(DataAdapter):
 
         full_prompt: str = context.strip() + self._generate_prompt(input_d=input_d, include_answer=False)
 
-        return {'data': [full_prompt], 'multi_choices': self.choices, 'system_prompt': self.system_prompt}
+        return self.gen_prompt_data(full_prompt)
 
     def get_gold_answer(self, input_d: dict) -> str:
         # Get the gold choice
@@ -100,26 +101,21 @@ class RACEAdapter(DataAdapter):
         Returns:
             The parsed answer. Depending on the dataset. Usually a string for chat.
         """
-        if eval_type == EvalType.CHECKPOINT:
+        if self.model_adapter == OutputType.MULTIPLE_CHOICE:
             return result
-        elif eval_type == EvalType.SERVICE:
-            return ResponseParser.parse_first_option_with_choices(result, self.choices)  # TODO: to be checked !
-        elif eval_type == EvalType.CUSTOM:
-            return ResponseParser.parse_first_option_with_choices(result, self.choices)  # TODO: to be checked !
         else:
-            raise ValueError(f'Unknown eval_type: {eval_type}')
+            return ResponseParser.parse_first_option_with_choices(result, self.choices)
 
     def match(self, gold: str, pred: str) -> float:
         return exact_match(gold=gold, pred=pred)
 
-    @classmethod
-    def _generate_prompt(cls, input_d: dict, include_answer=True) -> str:
+    def _generate_prompt(self, input_d: dict, include_answer=True) -> str:
 
         input_choices: list = input_d['options']
 
         example: str = 'Article:\n{}\nQuestion:\n{}'.format(input_d['article'], input_d['question'])
-        for j in range(len(cls.choices)):
-            example += '\n{}. {}'.format(cls.choices[j], input_choices[j])
+        for j in range(len(self.choices)):
+            example += '\n{}. {}'.format(self.choices[j], input_choices[j])
 
         example += '\nAnswer:'
         if include_answer:

evalscope/benchmarks/simple_qa/simple_qa_adapter.py

@@ -0,0 +1,20 @@
+from evalscope.benchmarks import Benchmark, DataAdapter
+from evalscope.utils.logger import get_logger
+
+# flake8: noqa
+
+logger = get_logger()
+
+
+@Benchmark.register(
+    name='simple_qa',
+    pretty_name='SimpleQA',
+    dataset_id='AI-ModelScope/SimpleQA',
+    metric_list=['AverageAccuracy'],
+    few_shot_num=0,
+    train_split=None,
+    eval_split='test')
+class SimpleQAAdapter(DataAdapter):
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)

evalscope/benchmarks/super_gpqa/five_shot_prompt.txt

@@ -0,0 +1,89 @@
+Answer the following multiple choice question. There is only one correct answer. The last line of your response should be in the format 'Answer: $LETTER' (without quotes), where LETTER is one of A, B, C, D, E, F, G, H, I, or J.
+
+Question:
+A refracting telescope consists of two converging lenses separated by 100 cm. The eye-piece lens has a focal length of 20 cm. The angular magnification of the telescope is
+A) 10
+B) 40
+C) 6
+D) 25
+E) 15
+F) 50
+G) 30
+H) 4
+I) 5
+J) 20
+
+Answer: Let's think step by step. In a refracting telescope, if both lenses are converging, the focus of both lenses must be between the two lenses, and thus the focal lengths of the two lenses must add up to their separation. Since the focal length of one lens is 20 cm, the focal length of the other must be 80 cm. The magnification is the ratio of these two focal lengths, or 4.
+Answer: H.
+
+Question:
+Say the pupil of your eye has a diameter of 5 mm and you have a telescope with an aperture of 50 cm. How much more light can the telescope gather than your eye?
+A) 1000 times more
+B) 50 times more
+C) 5000 times more
+D) 500 times more
+E) 10000 times more
+F) 20000 times more
+G) 2000 times more
+H) 100 times more
+I) 10 times more
+J) N/A
+
+Answer: Let's think step by step. The amount of light a telescope can gather compared to the human eye is proportional to the area of its apertures. The area of a circle is given by the formula $A = \pi \left(\frac{{D}}{{2}}\right)^2$, where $D$ is the diameter. Therefore, the relative light-gathering power is calculated as:
+\[
+\frac{{\left(\frac{{50 \text{{ cm}}}}{{2}}\right)^2}}{{\left(\frac{{5 \text{{ mm}}}}{{2}}\right)^2}} = \frac{{\left(\frac{{50 \text{{ cm}}}}{{0.1 \text{{ cm}}}}\right)^2}}{{\left(\frac{{5 \text{{ mm}}}}{{0.1 \text{{ cm}}}}\right)^2}} = \frac{{500^2}}{{5^2}} = 10000.
+\]
+Answer: E.
+
+Question:
+Where do most short-period comets come from and how do we know?
+A) The Kuiper belt; short period comets tend to be in the plane of the solar system like the Kuiper belt.
+B) The asteroid belt; short period comets tend to come from random directions indicating a spherical distribution of comets called the asteroid belt.
+C) The asteroid belt; short period comets tend to be in the plane of the solar system just like the asteroid belt.
+D) The Oort cloud; short period comets have orbital periods similar to asteroids like Vesta and are found in the plane of the solar system just like the Oort cloud.
+E) The Oort Cloud; short period comets tend to come from random directions indicating a spherical distribution of comets called the Oort Cloud.
+F) The Oort cloud; short period comets tend to be in the plane of the solar system just like the Oort cloud.
+G) The asteroid belt; short period comets have orbital periods similar to asteroids like Vesta and are found in the plane of the solar system just like the asteroid belt.
+Answer: Let's think step by step. Most short-period comets originate from the Kuiper belt. This is deduced from the observation that these comets tend to follow orbits that lie in the plane of the solar system, similar to the distribution of objects in the Kuiper belt itself. Thus, the alignment of these cometary orbits with the ecliptic plane points to their Kuiper belt origin.
+Answer: A.
+
+Question:
+Colors in a soap bubble result from light
+A) dispersion
+B) deflection
+C) refraction
+D) reflection
+E) interference
+F) converted to a different frequency
+G) polarization
+H) absorption
+I) diffraction
+J) transmission
+
+Answer: Let's think step by step. The colorful patterns observed in a soap bubble are caused by the phenomenon of light interference. This occurs when light waves bounce between the two surfaces of the soap film, combining constructively or destructively based on their phase differences and the varying thickness of the film. These interactions result in vibrant color patterns due to variations in the intensity of different wavelengths of light.
+Answer: E.
+
+Question:
+A microwave oven is connected to an outlet, 120 V, and draws a current of 2 amps. At what rate is energy being used by the microwave oven?
+A) 240 W
+B) 120 W
+C) 10 W
+D) 480 W
+E) 360 W
+F) 200 W
+G) 30 W
+H) 150 W
+I) 60 W
+J) 300 W
+
+Answer: Let's think step by step. The rate of energy usage, known as power, in an electrical circuit is calculated by the product of voltage and current. For a microwave oven connected to a 120 V outlet and drawing a current of 2 amps, the power consumption can be calculated as follows:
+\[
+\text{{Power}} = \text{{Voltage}} \times \text{{Current}} = 120 \, \text{{V}} \times 2 \, \text{{A}} = 240 \, \text{{W}}.
+\]
+Therefore, the microwave oven uses energy at a rate of 240 watts.
+Answer: A.
+
+Question:
+{query}
+
+Answer: Let's think step by step.