evalscope 0.12.0__py3-none-any.whl → 0.13.0__py3-none-any.whl

evalscope/benchmarks/data_adapter.py

@@ -5,7 +5,9 @@ from abc import ABC, abstractmethod
 from collections import defaultdict
 from typing import Any, List, Optional, Union
 
+from evalscope.benchmarks.utils import PromptData, preprocess_decorator
 from evalscope.constants import DEFAULT_DATASET_CACHE_DIR, AnswerKeys, EvalType, HubType
+from evalscope.metrics.llm_judge import LLMJudge
 from evalscope.metrics.named_metrics import metric_registry
 from evalscope.report import Report, ReportGenerator
 from evalscope.utils.logger import get_logger
@@ -18,8 +20,10 @@ class DataAdapter(ABC):
     def __init__(self,
                  name: str,
                  dataset_id: str,
+                 model_adapter: str,
                  subset_list: list,
                  metric_list: List[str],
+                 llm_as_a_judge: bool = False,
                  few_shot_num: Optional[int] = 0,
                  train_split: Optional[str] = None,
                  eval_split: Optional[str] = None,
@@ -48,6 +52,7 @@ class DataAdapter(ABC):
         """
         self.name = name
         self.dataset_id = dataset_id
+        self.model_adapter = model_adapter
         self.subset_list = subset_list
         self.metric_list = metric_list
         self.few_shot_num = few_shot_num
@@ -58,7 +63,17 @@ class DataAdapter(ABC):
         self.query_template = query_template
         self.pretty_name = pretty_name
         self.config_kwargs = kwargs
+        self.llm_as_a_judge = llm_as_a_judge
         self.category_map = kwargs.get('category_map', {})
+        self.choices = kwargs.get('choices', None)
+
+    def __init_subclass__(cls, **kwargs):
+        super().__init_subclass__(**kwargs)
+
+        # find and decorate parse_pred_result method
+        if hasattr(cls, 'parse_pred_result'):
+            original_method = cls.parse_pred_result
+            cls.parse_pred_result = preprocess_decorator(original_method)
 
     def load(self,
              dataset_name_or_path: str = None,
@@ -78,11 +93,17 @@ class DataAdapter(ABC):
 
         # Try to load dataset from local disk
         if os.path.exists(dataset_name_or_path):
-            data_dict = self.load_from_disk(dataset_name_or_path, subset_list, work_dir, **kwargs)
+            logger.info(f'Loading dataset from local disk: {dataset_name_or_path}')
+            trust_remote_code = kwargs.pop('trust_remote_code', False)
+            data_dict = self.load_from_disk(
+                dataset_name_or_path, subset_list, work_dir, trust_remote_code=trust_remote_code, **kwargs)
         else:
-            data_dict = self.load_from_hub(dataset_name_or_path, subset_list, work_dir, **kwargs)
-        if len(data_dict) == 0 or len(next(iter(data_dict.values()))) == 0:
-            raise ValueError(f'Local dataset is empty: {dataset_name_or_path}')
+            logger.info(f'Loading dataset from hub: {dataset_name_or_path}')
+            trust_remote_code = kwargs.pop('trust_remote_code', True)
+            data_dict = self.load_from_hub(
+                dataset_name_or_path, subset_list, work_dir, trust_remote_code=trust_remote_code, **kwargs)
+        if len(data_dict) == 0:
+            raise ValueError(f'Dataset is empty: {dataset_name_or_path}')
         return data_dict
 
     def load_from_hub(self, dataset_name_or_path: str, subset_list: list, work_dir: str, **kwargs) -> dict:
@@ -91,8 +112,7 @@ class DataAdapter(ABC):
         datasets_hub: str = kwargs.pop('datasets_hub', HubType.MODELSCOPE)
         split_as_subset: bool = kwargs.pop('split_as_subset', False)
         # Load dataset from remote
-        logger.info(
-            f'Loading dataset from {datasets_hub}: > dataset_name: {dataset_name_or_path} > subsets: {subset_list}')
+        logger.info(f'Loading dataset: dataset_name: {dataset_name_or_path} > subsets: {subset_list}')
 
         data_dict = {}
         split_list = [split for split in [self.train_split, self.eval_split] if split is not None]
@@ -133,21 +153,7 @@ class DataAdapter(ABC):
         If you want to support local dataset, please rewrite this method in xxx_data_adapter.
         Use modelscope.msdatasets.MsDataset.load to load the dataset from local by default.
         """
-        from modelscope.msdatasets import MsDataset
-
-        logger.info(f'Loading dataset from work_dir: {work_dir}: > dataset_name: {dataset_name_or_path} > \
-                subsets: {subset_list}')
-        data_dict = {}
-        subset_list = subset_list or self.subset_list
-        split_list = [split for split in [self.train_split, self.eval_split] if split is not None]
-        for sub_name in subset_list:
-            data_dict[sub_name] = {}
-            # e.g. train: few-shot, test: target dataset to evaluate
-            for split in split_list:
-                dataset = MsDataset.load(
-                    dataset_name=dataset_name_or_path, subset_name=sub_name, split=split, cache_dir=work_dir, **kwargs)
-                data_dict[sub_name].update({split: dataset})
-        return data_dict
+        return self.load_from_hub(dataset_name_or_path, subset_list, work_dir, **kwargs)
 
     def reformat_subset(self, data_dict: dict, subset_key: str, format: str = '{}') -> dict:
         """
@@ -285,6 +291,12 @@ class DataAdapter(ABC):
         kwargs['metric_list'] = self.metric_list
         return ReportGenerator.gen_report(subset_score_map, report_name, **kwargs)
 
+    def gen_prompt_data(self, prompt: str, **kwargs) -> dict:
+        if not isinstance(prompt, list):
+            prompt = [prompt]
+        prompt_data = PromptData(data=prompt, multi_choices=self.choices, system_prompt=self.system_prompt)
+        return prompt_data.to_dict()
+
     def gen_prompt(self, input_d: dict, subset_name: str, few_shot_list: list, **kwargs) -> Any:
         """
         Generate model prompt from raw input, unify the prompt format for different datasets.
@@ -348,3 +360,29 @@ class DataAdapter(ABC):
             The match result. Usually a score (float) for chat/multiple-choice-questions.
         """
         raise NotImplementedError
+
+    def llm_match(self, gold: Any, pred: Any, judge: Optional[LLMJudge] = None, **kwargs) -> float:
+        """
+        Use LLM as a judge to evaluate the predicted answer against the gold answer.
+
+        Args:
+            gold (Any): The golden answer.
+            pred (Any): The predicted answer.
+
+        Returns:
+            The match result as a float score between 0 and 1.
+        """
+        # Default judge handling
+        if judge is None:
+            logger.warning('No judge LLM provided, please specify a judge LLM in the config.')
+            return 0
+
+        # Extract question from raw_input if available
+        raw_input = kwargs.get('raw_input', {})
+        question_keys = ['question', 'prompt', 'query', 'problem']
+        question = next((raw_input.get(key) for key in question_keys if raw_input.get(key)), None)
+
+        # Request judge and obtain score
+        prompt = judge.build_prompt(pred, gold, question)
+        score = judge(prompt)
+        return judge.get_score(score)

evalscope/benchmarks/data_collection/data_collection_adapter.py

@@ -15,7 +15,6 @@ logger = get_logger()
 @Benchmark.register(
     name='data_collection',
     dataset_id='',  # dataset_id need to be set
-    model_adapter=ChatGenerationModelAdapter,
     subset_list=['default'],
     metric_list=['AverageAccuracy'],
     few_shot_num=0,

evalscope/benchmarks/general_mcq/general_mcq_adapter.py

@@ -3,9 +3,8 @@ import csv
 import os
 
 from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.constants import EvalType
+from evalscope.constants import EvalType, OutputType
 from evalscope.metrics.metrics import exact_match
-from evalscope.models import MultiChoiceModelAdapter
 from evalscope.utils import ResponseParser
 from evalscope.utils.logger import get_logger
 
@@ -16,8 +15,10 @@ logger = get_logger()
 
 @Benchmark.register(
     name='general_mcq',
+    pretty_name='General MCQ',
     dataset_id='general_mcq',
-    model_adapter=MultiChoiceModelAdapter,
+    model_adapter=OutputType.MULTIPLE_CHOICE,
+    output_types=[OutputType.MULTIPLE_CHOICE, OutputType.GENERATION],
     subset_list=['default'],
     metric_list=['AverageAccuracy'],
     few_shot_num=0,
@@ -27,11 +28,11 @@ logger = get_logger()
     query_template='问题：{question}\n{choices}\n答案: {answer}\n\n')
 class GeneralMCQAdapter(DataAdapter):
 
-    choices = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J']
-
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
 
+        self.choices = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J']
+
     def load_from_disk(self, dataset_name_or_path, subset_list, work_dir, **kwargs) -> dict:
         data_dict = {}
         for subset_name in subset_list:
@@ -85,7 +86,7 @@ class GeneralMCQAdapter(DataAdapter):
 
         full_prompt = self.prompt_template.format(query=context)
 
-        return {'data': [full_prompt], 'multi_choices': self.choices, 'system_prompt': self.system_prompt}
+        return self.gen_prompt_data(full_prompt)
 
     def get_gold_answer(self, input_d: dict) -> str:
         # Get the gold choice
@@ -103,14 +104,10 @@ class GeneralMCQAdapter(DataAdapter):
         Returns:
             The parsed answer. Depending on the dataset. Usually a string for chat.
         """
-        if eval_type == EvalType.CHECKPOINT:
+        if self.model_adapter == OutputType.MULTIPLE_CHOICE:
             return result
-        elif eval_type == EvalType.SERVICE:
-            return ResponseParser.parse_first_option_with_choices(result, self.choices)
-        elif eval_type == EvalType.CUSTOM:
-            return ResponseParser.parse_first_option_with_choices(result, self.choices)
         else:
-            raise ValueError(f'Invalid eval_type: {eval_type}')
+            return ResponseParser.parse_first_option_with_choices(text=result, options=self.choices)
 
     def match(self, gold: str, pred: str) -> float:
         return exact_match(gold=gold, pred=pred)

evalscope/benchmarks/general_qa/general_qa_adapter.py

@@ -1,12 +1,10 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
-import glob
 import os.path
 from collections import defaultdict
 from typing import List
 
 from evalscope.benchmarks import Benchmark, DataAdapter
 from evalscope.metrics import bleu_ngram_one_sample, compute_rouge_score_one_sample_zh, mean
-from evalscope.models import ChatGenerationModelAdapter
 from evalscope.utils.io_utils import jsonl_to_list
 from evalscope.utils.logger import get_logger
 
@@ -16,9 +14,8 @@ logger = get_logger()
 @Benchmark.register(
     name='general_qa',
     dataset_id='general_qa',
-    model_adapter=ChatGenerationModelAdapter,
     subset_list=['default'],
-    metric_list=['AverageBLEU'],
+    metric_list=['AverageBLEU', 'AverageRouge'],
     few_shot_num=0,
     train_split=None,
     eval_split='test',
@@ -31,18 +28,31 @@ class GeneralQAAdapter(DataAdapter):
 
         super().__init__(**kwargs)
 
-    def load(self, **kwargs) -> dict:
+    def load(self, dataset_name_or_path: str = None, subset_list: list = None, **kwargs) -> dict:
+        dataset_name_or_path = dataset_name_or_path or self.dataset_id
+        subset_list = subset_list or self.subset_list
 
-        data_file_list = glob.glob(os.path.join(self.dataset_id, '*.jsonl'))
+        data_file_dict = defaultdict(str)
         data_list = []
 
+        # get data file path and subset name
+        if os.path.isdir(dataset_name_or_path):
+            for subset_name in subset_list:
+                data_file_dict[subset_name] = os.path.join(dataset_name_or_path, f'{subset_name}.jsonl')
+        elif os.path.isfile(dataset_name_or_path):
+            cur_subset_name = os.path.basename(dataset_name_or_path).split('.')[0]
+            data_file_dict[cur_subset_name] = dataset_name_or_path
+        else:
+            raise ValueError(f'Invalid dataset path: {dataset_name_or_path}')
+
+        # load data from local disk
         try:
-            for file_path in data_file_list:
+            for subset_name, file_path in data_file_dict.items():
                 data_list.extend(jsonl_to_list(file_path))
         except Exception as e:
             raise ValueError(f'Failed to load data from {self.dataset_id}, got error: {e}')
 
-        data_dict = {'default': {'test': data_list}}
+        data_dict = {subset_name: {'test': data_list} for subset_name in data_file_dict.keys()}
 
         return data_dict
 
@@ -65,7 +75,7 @@ class GeneralQAAdapter(DataAdapter):
 
         query = input_d.get('question', '') or input_d.get('query', '')
         prompt = self.prompt_template.format(query=query)
-        return {'data': [prompt], 'system_prompt': self.system_prompt}
+        return self.gen_prompt_data(prompt)
 
     def get_gold_answer(self, input_d: dict) -> str:
         """
@@ -100,10 +110,12 @@ class GeneralQAAdapter(DataAdapter):
 
         """
         res = dict()
-        rouge_dict = compute_rouge_score_one_sample_zh([pred], [gold])
-        bleu_dict = bleu_ngram_one_sample(pred, gold)
-        res.update(rouge_dict)
-        res.update(bleu_dict)
+        if 'AverageRouge' in self.metric_list:
+            rouge_dict = compute_rouge_score_one_sample_zh([pred], [gold])
+            res.update(rouge_dict)
+        if 'AverageBLEU' in self.metric_list:
+            bleu_dict = bleu_ngram_one_sample(pred, gold)
+            res.update(bleu_dict)
         return res
 
     def compute_metric(self, review_res_list: List[dict], **kwargs) -> List[dict]:
@@ -119,7 +131,10 @@ class GeneralQAAdapter(DataAdapter):
         """
         items = defaultdict(list)
         for scores in review_res_list:
-            for k, v in scores.items():
-                items[k].append(v)
+            if isinstance(scores, dict):
+                for k, v in scores.items():
+                    items[k].append(v)
+            else:
+                items['AverageAccuracy'].append(scores)
         # items = [(score, 1.0) for score in review_res_list]
         return [{'metric_name': k, 'score': mean(v), 'num': len(v)} for k, v in items.items()]

evalscope/benchmarks/gpqa/gpqa_adapter.py

@@ -3,15 +3,16 @@ import random
 import re
 
 from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.constants import EvalType
+from evalscope.constants import EvalType, OutputType
 from evalscope.metrics import exact_match
-from evalscope.models import ChatGenerationModelAdapter
 
 
 @Benchmark.register(
     name='gpqa',
+    pretty_name='GPQA',
     dataset_id='modelscope/gpqa',
-    model_adapter=ChatGenerationModelAdapter,
+    model_adapter=OutputType.GENERATION,
+    output_types=[OutputType.MULTIPLE_CHOICE, OutputType.GENERATION],
     subset_list=['gpqa_extended', 'gpqa_main', 'gpqa_diamond'],
     metric_list=['AveragePass@1'],
     few_shot_num=5,
@@ -27,8 +28,9 @@ class GPQAAdapter(DataAdapter):
         self.choices = ['A', 'B', 'C', 'D']
         if self.few_shot_num and self.few_shot_num > 0:
             self.prompt_prefix = 'Here are some example questions from experts. Answer the final question yourself, following the format of the previous questions exactly.\n'  # noqa: E501
-            self.prompt_prefix += open(os.path.join(os.path.dirname(__file__), 'chain_of_thought.txt'),
-                                       'r').read() + '\nQuestion: '
+            self.prompt_prefix += open(
+                os.path.join(os.path.dirname(__file__), 'chain_of_thought.txt'), 'r',
+                encoding='utf-8').read() + '\nQuestion: '
         else:
             self.prompt_prefix = 'What is the correct answer to this question:'
 
@@ -50,7 +52,7 @@ class GPQAAdapter(DataAdapter):
         query = self.prompt_prefix + f"{input_d['Question']}\n{self.__form_options(processed_input_d['choices'])}"  # noqa: E501
 
         prompt = self.prompt_template.format(query=query)
-        return {'data': [prompt], 'multi_choices': self.choices, 'system_prompt': self.system_prompt}
+        return self.gen_prompt_data(prompt)
 
     def __process_input(self, input_d: dict) -> dict:
 
@@ -94,7 +96,10 @@ class GPQAAdapter(DataAdapter):
         """
         Parse the predicted result and extract proper answer.
         """
-        return GPQAAdapter.get_multiple_choice_answer(result)
+        if self.model_adapter == OutputType.MULTIPLE_CHOICE:
+            return result
+        else:
+            return GPQAAdapter.get_multiple_choice_answer(result)
 
     def match(self, gold: str, pred: str) -> float:
         """

evalscope/benchmarks/gsm8k/gsm8k_adapter.py

@@ -6,7 +6,6 @@ import os
 import re
 
 from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.models import ChatGenerationModelAdapter
 from evalscope.utils.io_utils import jsonl_to_list
 from evalscope.utils.logger import get_logger
 
@@ -15,8 +14,8 @@ logger = get_logger()
 
 @Benchmark.register(
     name='gsm8k',
+    pretty_name='GSM8K',
     dataset_id='modelscope/gsm8k',
-    model_adapter=ChatGenerationModelAdapter,
     subset_list=['main'],
     metric_list=['AverageAccuracy'],
     few_shot_num=4,
@@ -76,7 +75,7 @@ class GSM8KAdapter(DataAdapter):
 
         full_prompt = context + self.prompt_template.format(query=input_d['question'])
 
-        return {'data': [full_prompt], 'system_prompt': self.system_prompt}
+        return self.gen_prompt_data(full_prompt)
 
     def get_gold_answer(self, input_d: dict) -> str:
         # Extract the gold answer from the input dict.

evalscope/benchmarks/hellaswag/hellaswag_adapter.py

@@ -4,9 +4,8 @@ import os
 import re
 
 from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.constants import EvalType
+from evalscope.constants import EvalType, OutputType
 from evalscope.metrics import exact_match
-from evalscope.models import ContinuationLogitsModelAdapter
 from evalscope.utils.io_utils import jsonl_to_list
 from evalscope.utils.logger import get_logger
 from evalscope.utils.utils import ResponseParser
@@ -18,20 +17,19 @@ logger = get_logger()
 
 @Benchmark.register(
     name='hellaswag',
+    pretty_name='HellaSwag',
     dataset_id='modelscope/hellaswag',
-    model_adapter=ContinuationLogitsModelAdapter,
+    model_adapter=OutputType.MULTIPLE_CHOICE,
+    output_types=[OutputType.MULTIPLE_CHOICE, OutputType.GENERATION],
     subset_list=['default'],
     metric_list=['AverageAccuracy'],
     few_shot_num=0,
     train_split='train',
     eval_split='validation',
-    prompt_template=
-    'Respond with the index of sentence that makes the most sense, chose from 0, 1, 2, 3, derive your final answer as `The answer is ...`.',  # noqa: E501
+    prompt_template='{query}',  # noqa: E501
 )
 class HellaSwagAdapter(DataAdapter):
 
-    choices = ['0', '1', '2', '3']
-
     def __init__(self, **kwargs):
 
         few_shot_num = kwargs.get('few_shot_num', 0)
@@ -40,6 +38,7 @@ class HellaSwagAdapter(DataAdapter):
             kwargs['few_shot_num'] = 0
 
         super().__init__(**kwargs)
+        self.choices = ['A', 'B', 'C', 'D']
 
     def load_from_disk(self, dataset_name_or_path, subset_list, work_dir, **kwargs) -> dict:
         data_dict = {}
@@ -85,15 +84,14 @@ class HellaSwagAdapter(DataAdapter):
             self._generate_prompt(input_d=sample, endings=endings, include_answer=True) for sample in few_shot_list
         ]
         context: str = '\n'.join(few_shot_prompts) + '\n'
-        context += self._generate_prompt(input_d=input_d, endings=endings, include_answer=False)
-
-        ctx_continuation_pair_list = [(context.strip(), ' ' + cont.strip()) for cont in endings]
+        query = context.strip() + self._generate_prompt(input_d=input_d, endings=endings, include_answer=False)
 
-        return {'data': ctx_continuation_pair_list, 'multi_choices': self.choices, 'system_prompt': self.system_prompt}
+        full_prompt = self.prompt_template.format(query=query)
+        return self.gen_prompt_data(full_prompt)
 
     def get_gold_answer(self, input_d: dict) -> str:
-        # Get the gold choice
-        return input_d['label']
+        # Get the gold choice from the label
+        return self.choices[int(input_d['label'])]
 
     def parse_pred_result(self, result: list, raw_input_d: dict = None, eval_type: str = EvalType.CHECKPOINT) -> str:
         """
@@ -107,34 +105,22 @@ class HellaSwagAdapter(DataAdapter):
         Returns:
             The parsed answer. Depending on the dataset. Usually a string for chat.
         """
-        if eval_type == EvalType.CHECKPOINT:
-            # answer: in the form of [-2.3, -4.5, ...], len of self.choices
-            result = np.array(result)
-            endings: list = [self._preprocess(ending) for ending in raw_input_d['endings']]
-            completion_len = np.array([float(len(i)) for i in endings])
-            best_choice_idx = np.argmax(result / completion_len)
-
-            return str(best_choice_idx)
-        elif eval_type == EvalType.SERVICE:
-            return ResponseParser.parse_first_option(result)
-        elif eval_type == EvalType.CUSTOM:
-            return ResponseParser.parse_first_option(result)
+        if self.model_adapter == OutputType.MULTIPLE_CHOICE:
+            return result
         else:
-            raise ValueError(f'Invalid eval_type: {eval_type}')
+            return ResponseParser.parse_first_option(result)
 
     def match(self, gold: str, pred: str) -> float:
         return exact_match(gold=str(gold), pred=str(pred))
 
-    @classmethod
-    def _preprocess(cls, text):
+    def _preprocess(self, text):
         text = text.strip()
         text = text.replace(' [title]', '. ')
         text = re.sub('\\[.*?\\]', '', text)
         text = text.replace('  ', ' ')
         return text
 
-    @classmethod
-    def _generate_prompt(cls, input_d: dict, endings: list, include_answer=True) -> str:
+    def _generate_prompt(self, input_d: dict, endings: list, include_answer=True) -> str:
         """
         Generate prompt for HellaSwag dataset.
 
@@ -148,7 +134,13 @@ class HellaSwagAdapter(DataAdapter):
         """
 
         ctx = input_d['ctx_a'] + ' ' + input_d['ctx_b'].capitalize()
-        example: str = cls._preprocess(input_d['activity_label'] + ': ' + ctx)
+        # example: str = cls._preprocess(input_d['activity_label'] + ': ' + ctx)
+        example: str = self._preprocess(ctx)
+
+        example += '\nQuestion: Which ending makes the most sense?'
+        for i, ending in enumerate(endings):
+            example += f'\n{self.choices[i]}. {ending}'
+        example += '\nYou may choose from A, B, C, D. Derive your final answer as `The answer is ...`.'
 
         if include_answer:
             example += '{}\n\n'.format(endings[int(input_d['label'])])

evalscope/benchmarks/humaneval/humaneval_adapter.py

@@ -2,7 +2,6 @@
 import re
 
 from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.models import ChatGenerationModelAdapter
 from evalscope.utils.logger import get_logger
 
 logger = get_logger()
@@ -13,14 +12,18 @@ logger = get_logger()
 
 @Benchmark.register(
     name='humaneval',
+    pretty_name='HumanEval',
     dataset_id='modelscope/humaneval',
-    model_adapter=ChatGenerationModelAdapter,
     subset_list=['openai_humaneval'],
     metric_list=['Pass@1'],
     few_shot_num=0,
     train_split=None,
     eval_split='test',
     prompt_template='Complete the following python code:\n{query}',
+    extra_params={
+        'num_workers': 4,
+        'timeout': 4
+    },
 )
 class HumanevalAdapter(DataAdapter):
     """
@@ -35,17 +38,17 @@ class HumanevalAdapter(DataAdapter):
             raise ImportError('Please install human_eval:'
                               'https://github.com/openai/human-eval/tree/master#installation , '
                               'Note that you need to enable the execution code in the human_eval/execution.py first.')
+        super().__init__(**kwargs)
 
+        extra_params = kwargs.get('extra_params', {})
         self.k = [1]
-        self.num_workers = 4
-        self.timeout = 4.0
+        self.num_workers = extra_params.get('num_workers', 4)
+        self.timeout = extra_params.get('timeout', 4)
 
         self.read_problems_func = stream_jsonl
         self.write_jsonl_func = write_jsonl
         self.eval_func = check_correctness
 
-        super().__init__(**kwargs)
-
     def load_from_disk(self, dataset_name_or_path, subset_list, work_dir, **kwargs) -> dict:
         data_dict = {}
         for subset_name in subset_list:
@@ -66,7 +69,7 @@ class HumanevalAdapter(DataAdapter):
         query = input_d['prompt']
         full_prompt = self.prompt_template.format(query=query)
 
-        return {'data': [full_prompt], 'system_prompt': self.system_prompt}
+        return self.gen_prompt_data(full_prompt)
 
     @classmethod
     def _postprocess(cls, text: str) -> str:

evalscope/benchmarks/ifeval/ifeval_adapter.py

@@ -5,13 +5,12 @@ from evalscope.benchmarks import Benchmark, DataAdapter
 from evalscope.benchmarks.ifeval.utils import process_results
 from evalscope.constants import EvalType
 from evalscope.metrics import Metric, mean, metric_registry
-from evalscope.models import ChatGenerationModelAdapter
 
 
 @Benchmark.register(
     name='ifeval',
+    pretty_name='IFEval',
     dataset_id='opencompass/ifeval',
-    model_adapter=ChatGenerationModelAdapter,
     subset_list=['default'],
     metric_list=[
         'prompt_level_strict_acc',
@@ -36,7 +35,7 @@ class IFEvalAdapter(DataAdapter):
         metric_registry.register(Metric(name='inst_level_loose_acc', object=mean))
 
     def gen_prompt(self, input_d: dict, subset_name: str, few_shot_list: list, **kwargs) -> Any:
-        return {'data': [input_d['prompt']], 'system_prompt': self.system_prompt}
+        return self.gen_prompt_data(input_d['prompt'])
 
     def get_gold_answer(self, input_d: dict) -> str:
         return input_d

evalscope/benchmarks/iquiz/iquiz_adapter.py

@@ -1,14 +1,15 @@
 from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.constants import EvalType
+from evalscope.constants import EvalType, OutputType
 from evalscope.metrics import exact_match
-from evalscope.models import ChatGenerationModelAdapter
 from evalscope.utils.utils import ResponseParser
 
 
 @Benchmark.register(
     name='iquiz',
+    pretty_name='IQuiz',
     dataset_id='AI-ModelScope/IQuiz',
-    model_adapter=ChatGenerationModelAdapter,
+    model_adapter=OutputType.GENERATION,
+    output_types=[OutputType.MULTIPLE_CHOICE, OutputType.GENERATION],
     subset_list=['IQ', 'EQ'],
     metric_list=['AverageAccuracy'],
     few_shot_num=0,
@@ -36,7 +37,7 @@ class IQuizAdapter(DataAdapter):
         """
         prompt = f"问题: {input_d['question']}\n"
         prompt += self.__form_options(input_d['choices'])
-        return {'data': [prompt], 'multi_choices': self.choices, 'system_prompt': self.system_prompt}
+        return self.gen_prompt_data(prompt)
 
     def __form_options(self, options: list):
         option_str = '选项:\n'
@@ -54,7 +55,10 @@ class IQuizAdapter(DataAdapter):
         """
         Parse the predicted result and extract proper answer.
         """
-        return ResponseParser.parse_first_option_with_choices(result, self.choices)
+        if self.model_adapter == OutputType.MULTIPLE_CHOICE:
+            return result
+        else:
+            return ResponseParser.parse_first_option_with_choices(result, self.choices)
 
     def match(self, gold: str, pred: str) -> float:
         """