evalscope 0.11.0__py3-none-any.whl → 0.12.1__py3-none-any.whl

evalscope/arguments.py

@@ -1,7 +1,7 @@
 import argparse
 import json
 
-from evalscope.constants import EvalBackend, EvalStage, EvalType
+from evalscope.constants import EvalBackend, EvalStage, EvalType, OutputType
 
 
 class ParseStrArgsAction(argparse.Action):
@@ -71,6 +71,8 @@ def add_argument(parser: argparse.ArgumentParser):
     parser.add_argument('--seed', type=int, default=42, help='Random seed for reproducibility.')
     parser.add_argument('--api-key', type=str, default='EMPTY', help='The API key for the remote API model.')
     parser.add_argument('--api-url', type=str, default=None, help='The API url for the remote API model.')
+    parser.add_argument('--timeout', type=float, default=None, help='The timeout for the remote API model.')
+    parser.add_argument('--stream', action='store_true', default=False, help='Stream mode.')  # noqa: E501
     # yapf: enable
 
 

evalscope/benchmarks/{aime24 → aime}/aime24_adapter.py

@@ -1,6 +1,6 @@
 from evalscope.benchmarks import Benchmark, DataAdapter
+from evalscope.constants import OutputType
 from evalscope.metrics.math_parser import extract_answer, math_equal, strip_answer_string
-from evalscope.models import ChatGenerationModelAdapter
 from evalscope.utils.logger import get_logger
 
 # flake8: noqa
@@ -10,8 +10,8 @@ logger = get_logger()
 
 @Benchmark.register(
     name='aime24',
+    pretty_name='AIME-2024',
     dataset_id='HuggingFaceH4/aime_2024',
-    model_adapter=ChatGenerationModelAdapter,
     subset_list=['default'],
     metric_list=['AveragePass@1'],
     few_shot_num=0,
@@ -31,7 +31,7 @@ class AIME24Adapter(DataAdapter):
         problem = input_d['problem']
         full_prompt = self.prompt_template.format(query=problem)
 
-        return {'data': [full_prompt], 'system_prompt': self.system_prompt}
+        return self.gen_prompt_data(full_prompt)
 
     def get_gold_answer(self, input_d: dict) -> str:
         # Extract the gold answer from the input dict.

evalscope/benchmarks/aime/aime25_adapter.py

@@ -0,0 +1,49 @@
+from evalscope.benchmarks import Benchmark, DataAdapter
+from evalscope.constants import OutputType
+from evalscope.metrics.math_parser import extract_answer, math_equal, strip_answer_string
+from evalscope.utils.logger import get_logger
+
+# flake8: noqa
+
+logger = get_logger()
+
+
+@Benchmark.register(
+    name='aime25',
+    pretty_name='AIME-2025',
+    dataset_id='TIGER-Lab/AIME25',
+    subset_list=['default'],
+    metric_list=['AveragePass@1'],
+    few_shot_num=0,
+    train_split=None,
+    eval_split='train',  # Only train set is available
+    prompt_template='{query}\nPlease reason step by step, and put your final answer within \\boxed{{}}.',
+)
+class AIME25Adapter(DataAdapter):
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+    def gen_prompt(self, input_d: dict, few_shot_list: list, **kwargs) -> dict:
+        """
+        Generate the prompt for the model input.
+        """
+        problem = input_d['question']
+        full_prompt = self.prompt_template.format(query=problem)
+
+        return self.gen_prompt_data(full_prompt)
+
+    def get_gold_answer(self, input_d: dict) -> str:
+        # Extract the gold answer from the input dict.
+        return strip_answer_string(input_d['answer'])
+
+    def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = 'checkpoint') -> str:
+        """
+        Parse the model output to get the answer. Could be the best choice index.
+        """
+        # Note: Use same extraction method for both of checkpoint/service/custom
+        result = strip_answer_string(extract_answer(result))
+        return result
+
+    def match(self, gold: str, pred: str) -> float:
+        return math_equal(pred, gold)

evalscope/benchmarks/arc/arc_adapter.py

@@ -4,9 +4,8 @@ import json
 import os
 
 from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.constants import EvalType
+from evalscope.constants import EvalType, OutputType
 from evalscope.metrics import exact_match
-from evalscope.models import MultiChoiceModelAdapter
 from evalscope.utils import ResponseParser
 from evalscope.utils.logger import get_logger
 
@@ -17,19 +16,20 @@ logger = get_logger()
 
 @Benchmark.register(
     name='arc',
+    pretty_name='ARC',
     dataset_id='modelscope/ai2_arc',
-    model_adapter=MultiChoiceModelAdapter,
+    model_adapter=OutputType.MULTIPLE_CHOICE,
+    output_types=[OutputType.MULTIPLE_CHOICE, OutputType.GENERATION],
     subset_list=['ARC-Easy', 'ARC-Challenge'],
     metric_list=['AverageAccuracy'],
     few_shot_num=0,
     train_split='train',
     eval_split='test',
-    prompt_template='',
+    prompt_template=
+    'The following are multiple choice questions, please output correct answer in the form of A or B or C or D, do not output explanation:\n{query}',
 )
 class ARCAdapter(DataAdapter):
 
-    choices = ['A', 'B', 'C', 'D']
-
     def __init__(self, **kwargs):
         few_shot_num = kwargs.get('few_shot_num', None)
         if few_shot_num is None:
@@ -42,6 +42,8 @@ class ARCAdapter(DataAdapter):
 
         super().__init__(**kwargs)
 
+        self.choices = ['A', 'B', 'C', 'D']
+
     def load_from_disk(self, dataset_name_or_path, subset_list, work_dir, **kwargs) -> dict:
         """
         Load the dataset from local disk.
@@ -60,7 +62,7 @@ class ARCAdapter(DataAdapter):
             for split_name in ['Train', 'Test']:
                 split_path = os.path.join(subset_path, f'{subset_name}-{split_name}.jsonl')
                 if os.path.exists(split_path):
-                    with open(split_path, 'r', errors='ignore') as in_f:
+                    with open(split_path, 'r', errors='ignore', encoding='utf-8') as in_f:
                         rows = []
                         for line in in_f:
                             item = json.loads(line.strip())
@@ -107,12 +109,11 @@ class ARCAdapter(DataAdapter):
             {'data': ['xxx'], 'multi_choices': ['A', 'B', 'C', 'D']}
         """
         few_shot_prompts = [self._generate_prompt(input_d=sample, include_answer=True) for sample in few_shot_list]
-        context: str = '\n'.join(few_shot_prompts)
+        context = '\n'.join(few_shot_prompts) + self._generate_prompt(input_d=input_d, include_answer=False)
 
-        # context = f'The following are multiple choice questions, please output correct answer in the form of A or B or C or D, do not output explanation:\n {context}'
-        full_prompt: str = context + self._generate_prompt(input_d=input_d, include_answer=False)
+        full_prompt = self.prompt_template.format(query=context)
 
-        return {'data': [full_prompt], 'multi_choices': self.choices, 'system_prompt': self.system_prompt}
+        return self.gen_prompt_data(full_prompt)
 
     def get_gold_answer(self, input_d: dict) -> str:
         # Get the gold choice
@@ -130,14 +131,10 @@ class ARCAdapter(DataAdapter):
         Returns:
             The parsed answer. Depending on the dataset. Usually a string for chat.
         """
-        if eval_type == EvalType.CHECKPOINT:
+        if self.model_adapter == OutputType.MULTIPLE_CHOICE:
             return result
-        elif eval_type == EvalType.SERVICE:
-            return ResponseParser.parse_first_option_with_choices(text=result, options=self.choices)
-        elif eval_type == EvalType.CUSTOM:
-            return ResponseParser.parse_first_option_with_choices(text=result, options=self.choices)
         else:
-            raise ValueError(f'Invalid eval_type: {eval_type}')
+            return ResponseParser.parse_first_capital(text=result, options=self.choices)
 
     def match(self, gold: str, pred: str) -> float:
         return exact_match(gold=gold, pred=pred)

evalscope/benchmarks/bbh/bbh_adapter.py

@@ -8,8 +8,6 @@ import re
 from evalscope.benchmarks import Benchmark, DataAdapter
 from evalscope.constants import AnswerKeys
 from evalscope.metrics import exact_match
-from evalscope.models.chat_adapter import ChatGenerationModelAdapter
-from evalscope.utils import ResponseParser
 from evalscope.utils.logger import get_logger
 
 # flake8: noqa
@@ -60,8 +58,8 @@ SUBSET_LIST = MULTIPLE_CHOICE_LIST + FREE_FORM_LIST
 
 @Benchmark.register(
     name='bbh',
+    pretty_name='BBH',
     dataset_id='modelscope/bbh',
-    model_adapter=ChatGenerationModelAdapter,
     subset_list=SUBSET_LIST,
     metric_list=['AverageAccuracy'],
     few_shot_num=3,
@@ -94,7 +92,7 @@ class BBHAdapter(DataAdapter):
                 else:
                     file_path: str = os.path.join(work_dir, dataset_name_or_path, f'{subset_name}.json')
                 if os.path.exists(file_path):
-                    with open(file_path, 'r') as f:
+                    with open(file_path, 'r', encoding='utf-8') as f:
                         examples = json.load(f)['examples']
                         if subset_name in data_dict:
                             data_dict[subset_name].update({split_name: examples})
@@ -125,7 +123,7 @@ class BBHAdapter(DataAdapter):
             cot_prompts = ''
         full_prompt = cot_prompts + self.prompt_template.format(query=input_d['input'])
 
-        return {'data': [full_prompt], 'system_prompt': self.system_prompt}
+        return self.gen_prompt_data(full_prompt)
 
     def gen_prompts(self, data_dict: dict) -> dict:
         """
@@ -153,7 +151,9 @@ class BBHAdapter(DataAdapter):
         for sub_name, sub_data_dict in data_dict.items():
             few_shot_data = []
             if self.few_shot_num > 0:
-                with open(os.path.join(os.path.dirname(__file__), 'cot_prompts', f'{sub_name}.txt'), 'r') as f:
+                with open(
+                        os.path.join(os.path.dirname(__file__), 'cot_prompts', f'{sub_name}.txt'), 'r',
+                        encoding='utf-8') as f:
                     cot_prompt_str = f.read()
                 few_shot_data = [cot_prompt_str]
 
@@ -171,11 +171,6 @@ class BBHAdapter(DataAdapter):
                 prompt_d[AnswerKeys.RAW_INPUT] = sample_d_new
                 res_dict[sub_name].append(prompt_d)
 
-        rnd = random.Random()
-        rnd.seed(42)
-        for k, v in res_dict.items():
-            rnd.shuffle(v)
-
         return res_dict
 
     def get_gold_answer(self, input_d: dict) -> str:

evalscope/benchmarks/benchmark.py

@@ -1,12 +1,13 @@
 import copy
+from collections import OrderedDict
 from dataclasses import dataclass, field
 from typing import TYPE_CHECKING, Dict, List, Optional
 
+from evalscope.constants import OutputType
+
 if TYPE_CHECKING:
     from evalscope.benchmarks import DataAdapter
 
-from evalscope.models import BaseModelAdapter
-
 BENCHMARK_MAPPINGS = {}
 
 
@@ -15,8 +16,9 @@ class BenchmarkMeta:
     name: str
     dataset_id: str
     data_adapter: 'DataAdapter'
-    model_adapter: BaseModelAdapter
-    subset_list: List[str] = field(default_factory=list)
+    model_adapter: Optional[str] = OutputType.GENERATION
+    output_types: Optional[List[str]] = field(default_factory=lambda: [OutputType.GENERATION])
+    subset_list: List[str] = field(default_factory=lambda: ['default'])
     metric_list: List[str] = field(default_factory=list)
     few_shot_num: int = 0
     few_shot_random: bool = False
@@ -24,6 +26,9 @@ class BenchmarkMeta:
     eval_split: Optional[str] = None
     prompt_template: Optional[str] = None
     system_prompt: Optional[str] = None
+    query_template: Optional[str] = None
+    pretty_name: Optional[str] = None
+    filters: Optional[OrderedDict] = None
 
     def _update(self, args: dict):
         if args.get('local_path'):
@@ -37,10 +42,7 @@ class BenchmarkMeta:
     def to_string_dict(self) -> dict:
         cur_dict = copy.deepcopy(self.__dict__)
         # cur_dict['data_adapter'] = self.data_adapter.__name__
-        # cur_dict['model_adapter'] = self.model_adapter.__name__
-        # cur_dict['metric_list'] = [metric['name'] for metric in self.metric_list]
         del cur_dict['data_adapter']
-        del cur_dict['model_adapter']
         return cur_dict
 
     def get_data_adapter(self, config: dict = {}) -> 'DataAdapter':
@@ -59,18 +61,18 @@ class Benchmark:
     @classmethod
     def get(cls, name: str) -> 'BenchmarkMeta':
         if name not in BENCHMARK_MAPPINGS:
-            raise Exception(f'Unknown benchmark: {name}. Available tasks: {BENCHMARK_MAPPINGS.keys()}')
+            raise Exception(f'Unknown benchmark: {name}. Available tasks: {list(BENCHMARK_MAPPINGS.keys())}')
         benchmark = BENCHMARK_MAPPINGS[name]
         return benchmark
 
     @classmethod
-    def register(cls, name: str, dataset_id: str, model_adapter: BaseModelAdapter, **kwargs):
+    def register(cls, name: str, dataset_id: str, **kwargs):
 
         def register_wrapper(data_adapter):
             if name in BENCHMARK_MAPPINGS:
                 raise Exception(f'Benchmark {name} already registered')
             BENCHMARK_MAPPINGS[name] = BenchmarkMeta(
-                name=name, data_adapter=data_adapter, model_adapter=model_adapter, dataset_id=dataset_id, **kwargs)
+                name=name, data_adapter=data_adapter, dataset_id=dataset_id, **kwargs)
             return data_adapter
 
         return register_wrapper

evalscope/benchmarks/ceval/ceval_adapter.py

@@ -3,9 +3,8 @@ import csv
 import os
 
 from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.constants import EvalType
+from evalscope.constants import EvalType, OutputType
 from evalscope.metrics.metrics import exact_match
-from evalscope.models import MultiChoiceModelAdapter
 from evalscope.utils import ResponseParser
 from evalscope.utils.logger import get_logger
 
@@ -126,8 +125,10 @@ SUBJECT_MAPPING = {
 
 @Benchmark.register(
     name='ceval',
+    pretty_name='C-Eval',
     dataset_id='modelscope/ceval-exam',
-    model_adapter=MultiChoiceModelAdapter,
+    model_adapter=OutputType.MULTIPLE_CHOICE,
+    output_types=[OutputType.MULTIPLE_CHOICE, OutputType.GENERATION],
     subset_list=SUBSET_LIST,
     metric_list=['AverageAccuracy'],
     few_shot_num=0,
@@ -137,8 +138,6 @@ SUBJECT_MAPPING = {
 )
 class CEVALAdapter(DataAdapter):
 
-    choices = ['A', 'B', 'C', 'D']
-
     def __init__(self, **kwargs):
 
         few_shot_num = kwargs.get('few_shot_num', 0)
@@ -148,6 +147,7 @@ class CEVALAdapter(DataAdapter):
         super().__init__(**kwargs)
 
         self.category_map = {k: v[-1] for k, v in SUBJECT_MAPPING.items()}
+        self.choices = ['A', 'B', 'C', 'D']
 
     def load_from_disk(self, dataset_name_or_path, subset_list, work_dir, **kwargs) -> dict:
         data_dict = {}
@@ -207,7 +207,7 @@ class CEVALAdapter(DataAdapter):
         subject_name: str = SUBJECT_MAPPING.get(subset_name)[1] if SUBJECT_MAPPING.get(subset_name) else subset_name
         full_prompt = self.prompt_template.format(subset_name=subject_name, query=query)
 
-        return {'data': [full_prompt], 'multi_choices': self.choices, 'system_prompt': self.system_prompt}
+        return self.gen_prompt_data(full_prompt)
 
     def get_gold_answer(self, input_d: dict) -> str:
         # Get the gold choice
@@ -225,22 +225,17 @@ class CEVALAdapter(DataAdapter):
         Returns:
             The parsed answer. Depending on the dataset. Usually a string for chat.
         """
-        if eval_type == EvalType.CHECKPOINT:
+        if self.model_adapter == OutputType.MULTIPLE_CHOICE:
             return result
-        elif eval_type == EvalType.SERVICE:
-            return ResponseParser.parse_first_option_with_choices(result, self.choices)
-        elif eval_type == EvalType.CUSTOM:
-            return ResponseParser.parse_first_option_with_choices(result, self.choices)
         else:
-            raise ValueError(f'Invalid eval_type: {eval_type}')
+            return ResponseParser.parse_first_option_with_choices(text=result, options=self.choices)
 
     def match(self, gold: str, pred: str) -> float:
         return exact_match(gold=gold, pred=pred)
 
-    @classmethod
-    def _format_example(cls, input_d: dict, include_answer=True):
+    def _format_example(self, input_d: dict, include_answer=True):
         example = '问题：' + input_d['question']
-        for choice in cls.choices:
+        for choice in self.choices:
             example += f'\n{choice}. {input_d[f"{choice}"]}'
 
         if include_answer:

evalscope/benchmarks/cmmlu/cmmlu_adapter.py

@@ -4,9 +4,8 @@ import csv
 import os
 
 from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.constants import EvalType
+from evalscope.constants import EvalType, OutputType
 from evalscope.metrics import exact_match
-from evalscope.models import MultiChoiceModelAdapter
 from evalscope.utils import ResponseParser
 from evalscope.utils.logger import get_logger
 
@@ -103,8 +102,10 @@ SUBJECT_MAPPING = {
 
 @Benchmark.register(
     name='cmmlu',
+    pretty_name='C-MMLU',
     dataset_id='modelscope/cmmlu',
-    model_adapter=MultiChoiceModelAdapter,
+    model_adapter=OutputType.MULTIPLE_CHOICE,
+    output_types=[OutputType.MULTIPLE_CHOICE, OutputType.GENERATION],
     subset_list=SUBSET_LIST,
     metric_list=['AverageAccuracy'],
     few_shot_num=5,
@@ -114,12 +115,11 @@ SUBJECT_MAPPING = {
 )
 class CMMLUAdapter(DataAdapter):
 
-    choices = ['A', 'B', 'C', 'D']
-
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
 
         self.category_map = {k: v[-1] for k, v in SUBJECT_MAPPING.items()}
+        self.choices = ['A', 'B', 'C', 'D']
 
     def load_from_disk(self, dataset_name_or_path, subset_list, work_dir, **kwargs) -> dict:
         data_dict = {}
@@ -172,7 +172,7 @@ class CMMLUAdapter(DataAdapter):
 
         full_prompt = self.prompt_template.format(subset_name=self._format_subject(subset_name), query=context.strip())
 
-        return {'data': [full_prompt], 'multi_choices': self.choices, 'system_prompt': self.system_prompt}
+        return self.gen_prompt_data(full_prompt)
 
     def get_gold_answer(self, input_d: dict) -> str:
         # Get the gold choice
@@ -190,26 +190,21 @@ class CMMLUAdapter(DataAdapter):
         Returns:
             The parsed answer. Depending on the dataset. Usually a string for chat.
         """
-        if eval_type == EvalType.CHECKPOINT:
+        if self.model_adapter == OutputType.MULTIPLE_CHOICE:
             return result
-        elif eval_type == EvalType.SERVICE:
-            return ResponseParser.parse_first_option_with_choices(result, self.choices)
-        elif eval_type == EvalType.CUSTOM:
-            return ResponseParser.parse_first_option_with_choices(result, self.choices)
         else:
-            raise ValueError(f'Invalid eval_type: {eval_type}')
+            return ResponseParser.parse_first_option_with_choices(text=result, options=self.choices)
 
     def match(self, gold: str, pred: str) -> float:
         return exact_match(gold=gold, pred=pred)
 
-    @classmethod
-    def _generate_prompt(cls, input_d: dict, include_answer=True) -> str:
+    def _generate_prompt(self, input_d: dict, include_answer=True) -> str:
 
         input_choices: list = [input_d['A'], input_d['B'], input_d['C'], input_d['D']]
 
         example: str = input_d['Question']
-        for j in range(len(cls.choices)):
-            example += '\n{}. {}'.format(cls.choices[j], input_choices[j])
+        for j in range(len(self.choices)):
+            example += '\n{}. {}'.format(self.choices[j], input_choices[j])
 
         example += '\nAnswer:'
         if include_answer:

evalscope/benchmarks/competition_math/competition_math_adapter.py

@@ -18,12 +18,12 @@ logger = get_logger()
 
 @Benchmark.register(
     name='competition_math',
+    pretty_name='MATH',
     dataset_id='modelscope/competition_math',
-    model_adapter=ChatGenerationModelAdapter,
     subset_list=['Level 1', 'Level 2', 'Level 3', 'Level 4', 'Level 5'],
     metric_list=['AveragePass@1'],
     few_shot_num=4,
-    train_split='train',
+    train_split=None,
     eval_split='test',
     prompt_template='{query}\nPlease reason step by step, and put your final answer within \\boxed{{}}.',
 )
@@ -43,7 +43,8 @@ class CompetitionMathAdapter(DataAdapter):
     def load(self, **kwargs):
         # default load all levels
         kwargs['subset_list'] = ['default']
-        return super().load(**kwargs)
+        data_dict = super().load(**kwargs)
+        return self.reformat_subset(data_dict, subset_key='level')
 
     def load_from_disk(self, dataset_name_or_path, subset_list, work_dir, **kwargs) -> dict:
         data_dict = defaultdict(dict)
@@ -57,27 +58,12 @@ class CompetitionMathAdapter(DataAdapter):
                 split_data = []
                 for file_path in split_files:
                     if os.path.exists(file_path):
-                        with open(file_path, 'r') as f:
+                        with open(file_path, 'r', encoding='utf-8') as f:
                             split_data.append(json.load(f))
                 data_dict[subset_name][split_name] = split_data
 
         return data_dict
 
-    def gen_prompts(self, data_dict: dict) -> dict:
-        res_dict: dict = defaultdict(list)
-
-        #  use level as subset
-        for sub_name, sub_data_dict in data_dict.items():
-            for sample_d in sub_data_dict[self.eval_split]:
-                level = sample_d['level']
-                if level not in self.subset_list:
-                    continue
-                prompt_d = self.gen_prompt(input_d=sample_d, few_shot_list=None)
-                prompt_d[AnswerKeys.RAW_INPUT] = sample_d
-                res_dict[level].append(prompt_d)
-
-        return res_dict
-
     def gen_prompt(self, input_d: dict, few_shot_list: list, **kwargs) -> dict:
         """
         Generate the prompt for the model input.
@@ -95,7 +81,7 @@ class CompetitionMathAdapter(DataAdapter):
         use_fewshot = self.few_shot_num > 0
         query = self._generate_prompt(input_d, use_fewshot=use_fewshot)
         full_prompt = self.prompt_template.format(query=query)
-        return {'data': [full_prompt], 'system_prompt': self.system_prompt}
+        return self.gen_prompt_data(full_prompt)
 
     def get_gold_answer(self, input_d: dict) -> str:
         # Extract the gold answer from the input dict.