evalscope 0.16.0__py3-none-any.whl → 0.16.2__py3-none-any.whl

evalscope/benchmarks/aime/aime25_adapter.py

@@ -1,5 +1,4 @@
 from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.constants import OutputType
 from evalscope.metrics import extract_answer, math_equal, strip_answer_string
 from evalscope.utils.logger import get_logger
 
@@ -11,6 +10,9 @@ logger = get_logger()
 @Benchmark.register(
     name='aime25',
     pretty_name='AIME-2025',
+    tags=['Mathematics'],
+    description=
+    'The AIME 2025 benchmark is based on problems from the American Invitational Mathematics Examination, a prestigious high school mathematics competition. This benchmark tests a model’s ability to solve challenging mathematics problems by generating step-by-step solutions and providing the correct final answer.',
     dataset_id='opencompass/AIME2025',
     subset_list=['AIME2025-I', 'AIME2025-II'],
     metric_list=['AveragePass@1'],

evalscope/benchmarks/alpaca_eval/alpaca_eval_adapter.py

@@ -47,6 +47,11 @@ Evaluate the models based on the quality and relevance of their outputs, and sel
 @Benchmark.register(
     name='alpaca_eval',
     pretty_name='AlpacaEval2.0',
+    tags=['Instruction-Following', 'Reasoning'],
+    description='Alpaca Eval 2.0 is an enhanced framework for evaluating instruction-following language models, '
+    'featuring an improved auto-annotator, updated baselines, and continuous preference calculation to '
+    'provide more accurate and cost-effective model assessments. '
+    'Currently not support `length-controlled winrate`; the official Judge model is `gpt-4-1106-preview`, while the baseline model is `gpt-4-turbo`.',  # noqa: E501
     dataset_id='AI-ModelScope/alpaca_eval',
     subset_list=['alpaca_eval_gpt4_baseline'],
     metric_list=['winrate'],

evalscope/benchmarks/arc/arc_adapter.py

@@ -17,6 +17,9 @@ logger = get_logger()
 @Benchmark.register(
     name='arc',
     pretty_name='ARC',
+    tags=['Reasoning', 'MCQ'],
+    description=
+    'The ARC (AI2 Reasoning Challenge) benchmark is designed to evaluate the reasoning capabilities of AI models through multiple-choice questions derived from science exams. It includes two subsets: ARC-Easy and ARC-Challenge, which vary in difficulty.',  # noqa: E501
     dataset_id='modelscope/ai2_arc',
     model_adapter=OutputType.GENERATION,
     output_types=[OutputType.MULTIPLE_CHOICE, OutputType.GENERATION],

evalscope/benchmarks/arena_hard/arena_hard_adapter.py

@@ -1,5 +1,3 @@
-import re
-from collections import defaultdict
 from typing import Any, List
 
 from evalscope.benchmarks import Benchmark, DataAdapter
@@ -19,12 +17,18 @@ GRADER_TEMPLATE = "<|User Prompt|>\n{question}\n\n<|The Start of Assistant A's A
 @Benchmark.register(
     name='arena_hard',
     pretty_name='ArenaHard',
+    tags=['Instruction-Following', 'Reasoning'],
+    description=
+    'ArenaHard is a benchmark designed to evaluate the performance of large language models in a competitive setting, '
+    'where models are pitted against each other in a series of tasks to determine their relative strengths and weaknesses. '
+    'It includes a set of challenging tasks that require reasoning, understanding, and generation capabilities. '
+    'Currently not support `style-controlled winrate`; the official Judge model is `gpt-4-1106-preview`, while the baseline model is `gpt-4-0314`.',  # noqa: E501
     dataset_id='AI-ModelScope/arena-hard-auto-v0.1',
     metric_list=['winrate'],
     few_shot_num=0,
     train_split=None,
     eval_split='test')
-class AlpacaEvalAdapter(DataAdapter):
+class ArenaHardAdapter(DataAdapter):
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)

evalscope/benchmarks/bbh/bbh_adapter.py

@@ -59,6 +59,9 @@ SUBSET_LIST = MULTIPLE_CHOICE_LIST + FREE_FORM_LIST
 @Benchmark.register(
     name='bbh',
     pretty_name='BBH',
+    tags=['Reasoning'],
+    description=
+    'The BBH (Big Bench Hard) benchmark is a collection of challenging tasks designed to evaluate the reasoning capabilities of AI models. It includes both free-form and multiple-choice tasks, covering a wide range of reasoning skills.',  # noqa: E501
     dataset_id='modelscope/bbh',
     subset_list=SUBSET_LIST,
     metric_list=['AverageAccuracy'],

evalscope/benchmarks/benchmark.py

@@ -28,6 +28,8 @@ class BenchmarkMeta:
     system_prompt: Optional[str] = None
     query_template: Optional[str] = None
     pretty_name: Optional[str] = None
+    description: Optional[str] = None
+    tags: Optional[List[str]] = field(default_factory=list)
     filters: Optional[OrderedDict] = None
     extra_params: Optional[Dict] = field(default_factory=dict)
 

evalscope/benchmarks/bfcl/bfcl_adapter.py

@@ -0,0 +1,237 @@
+import copy
+import importlib
+import json
+import re
+import traceback
+from typing import Any, List
+
+from evalscope.benchmarks import Benchmark, DataAdapter
+from evalscope.constants import EvalType
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+SUBJECT_MAPPING = {
+    'simple': 'AST_NON_LIVE',
+    'multiple': 'AST_NON_LIVE',
+    'parallel': 'AST_NON_LIVE',
+    'parallel_multiple': 'AST_NON_LIVE',
+    'java': 'AST_NON_LIVE',
+    'javascript': 'AST_NON_LIVE',
+    'live_simple': 'AST_LIVE',
+    'live_multiple': 'AST_LIVE',
+    'live_parallel': 'AST_LIVE',
+    'live_parallel_multiple': 'AST_LIVE',
+    'irrelevance': 'RELEVANCE',
+    'live_relevance': 'RELEVANCE',
+    'live_irrelevance': 'RELEVANCE',
+    'multi_turn_base': 'MULTI_TURN',
+    'multi_turn_miss_func': 'MULTI_TURN',
+    'multi_turn_miss_param': 'MULTI_TURN',
+    'multi_turn_long_context': 'MULTI_TURN'
+}
+
+
+@Benchmark.register(
+    name='bfcl_v3',
+    pretty_name='BFCL-v3',
+    tags=['Agent'],
+    description=
+    'Berkeley Function Calling Leaderboard (BFCL), the **first comprehensive and executable function call evaluation** '
+    'dedicated to assessing Large Language Models\' (LLMs) ability to invoke functions. Unlike previous evaluations, '
+    'BFCL accounts for various forms of function calls, diverse scenarios, and executability. '
+    'Need to run `pip install bfcl-eval` before evaluating. '
+    '[Usage Example](https://evalscope.readthedocs.io/zh-cn/latest/third_party/bfcl_v3.html)',  # noqa: E501
+    dataset_id='AI-ModelScope/bfcl_v3',
+    subset_list=list(SUBJECT_MAPPING.keys()),
+    model_adapter='bfcl_server',
+    metric_list=['AverageAccuracy'],
+    few_shot_num=0,
+    train_split=None,
+    eval_split='train',
+    extra_params={
+        'underscore_to_dot': True,
+        'is_fc_model': True,
+    })
+class BFCLAdapter(DataAdapter):
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+        spec = importlib.util.find_spec('bfcl_eval')
+        if spec is None:
+            raise ImportError(
+                '`bfcl_eval` not found, please install it with `pip install bfcl-eval` before evaluating.')
+
+        self.category_map = SUBJECT_MAPPING
+
+        extra_params = kwargs.get('extra_params', {})
+        self.underscore_to_dot = extra_params.get('underscore_to_dot', False)
+        self.is_fc_model = extra_params.get('is_fc_model', True)
+
+    def load(self, **kwargs):
+        kwargs['subset_list'] = ['default']
+        data_dict = super().load(**kwargs)
+        return self.reformat_subset(data_dict, subset_key='subset', format='{}')
+
+    def preprocess_row(self, row: dict):
+        """
+        Inplace preprocess the row to ensure it has the correct format for BFCL evaluation.
+        """
+        row['should_execute_tool_calls'] = True if row['multi_turn'] else False
+        row['functions'] = json.loads(row['functions'])
+        row['tools'] = json.loads(row['tools'])
+        row['turns'] = json.loads(row['turns'])
+        row['missing_functions'] = json.loads(row['missed_functions'])
+        row['ground_truth'] = json.loads(row.get('ground_truth', '{}'))
+        row['initial_config'] = json.loads(row['initial_config'])
+        row['is_fc_model'] = self.is_fc_model
+
+    def gen_prompt(self, input_d, subset_name, few_shot_list, **kwargs):
+        self.preprocess_row(input_d)
+
+        # If the model is a function calling model, we need to remove the system prompt
+        if self.is_fc_model:
+            turns = input_d['turns']
+            new_turns = []
+            for turn_idx, messages in enumerate(turns):
+                current_messages = messages.copy()
+                if len(current_messages) > 0 and current_messages[0]['role'] == 'system':
+                    current_messages = current_messages[1:]
+                new_turns.append(current_messages)
+            input_d['turns'] = new_turns
+
+        return self.gen_prompt_data(prompt='', messages=input_d)
+
+    def get_gold_answer(self, input_d: dict) -> str:
+        # Get the gold choice
+        return input_d.get('ground_truth', )
+
+    def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = EvalType.CHECKPOINT) -> dict:
+        row = copy.deepcopy(raw_input_d)
+        del row['turns']  # Remove turns as they are not needed for the match function
+
+        row['generation'] = result
+        return row
+
+    def match(self, gold: dict, pred: dict) -> dict:
+        from bfcl_eval.eval_checker.ast_eval.ast_checker import ast_checker
+        from bfcl_eval.eval_checker.multi_turn_eval.multi_turn_checker import multi_turn_checker
+        from bfcl_eval.model_handler.utils import (convert_to_function_call, default_decode_ast_prompting,
+                                                   default_decode_execute_prompting)
+        from bfcl_eval.utils import is_empty_output
+
+        # NOTE: This is hardcoded dummy model since its only use is to infer underscore_to_dot
+        # which decides if model was provided with functions of the type
+        # spotify.list_songs or spotify_list_songs
+        # It is False for all llama models (when using via prompting)
+        # and True for API calls
+        if self.underscore_to_dot:
+            dummy_model = 'gpt-4o-2024-11-20-FC'
+        else:
+            dummy_model = 'meta-llama/Llama-3.3-70B-Instruct-FC'
+
+        row = pred
+        test_category = re.sub(r'_[0-9_-]+$', '', row['id'])
+        if test_category in {'irrelevance', 'live_irrelevance', 'live_relevance'}:
+            error = None
+            try:
+                if self.is_fc_model:
+                    decoded_tool_calls = []
+                    for tool_call in row['generation'][0]:
+                        name = list(tool_call.keys())[0]
+                        params = json.loads(tool_call[name])
+                        decoded_tool_calls.append({name: params})
+                else:
+                    decoded_tool_calls = default_decode_ast_prompting(row['generation'][0][0], row['language'])
+
+                # successful decode means valid function call was present
+                contains_func_call = True
+                if is_empty_output(decoded_tool_calls):
+                    # Empty output is not considered as a valid function call
+                    contains_func_call = False
+                    error = 'Empty decoded output.'
+            except Exception:
+                contains_func_call = False
+                error = f'Failed to decode with traceback: {traceback.format_exc()}'
+            finally:
+                valid = contains_func_call if test_category == 'live_relevance' else not contains_func_call
+                score_result = {'valid': valid, 'error_message': error}
+
+        elif row['multi_turn']:
+            # each step might give a list of tool calls and each turn is multi-step
+            # and multi-turn has generations of all the turns
+            # hence in a multi-turn setting,
+            # multi_turn_decoded_generations is a list of list of list of strings
+            multi_turn_decoded_generations: list[list[list[str]]] = []
+            for single_turn_generations in row['generation']:
+                single_turn_decoded_generations: list[list[str]] = []
+                for generation in single_turn_generations:
+                    try:
+                        if self.is_fc_model:
+                            tool_calls = convert_to_function_call(generation)
+                        else:
+                            tool_calls = default_decode_execute_prompting(generation)
+
+                        single_turn_decoded_generations.append(tool_calls)
+                    except Exception:
+                        single_turn_decoded_generations.append([generation])
+
+                multi_turn_decoded_generations.append(single_turn_decoded_generations)
+
+            try:
+                raw_score_result = multi_turn_checker(
+                    multi_turn_decoded_generations,
+                    row['ground_truth'],
+                    row,
+                    test_category,
+                    dummy_model,
+                )
+            except Exception:
+                raw_score_result = {
+                    'valid': False,
+                    'error_type': 'multi_turn:checker_failed',
+                    'error_message': f'Failed to grade multi-turn. Traceback: {traceback.format_exc()}',
+                }
+
+            score_result = {
+                'valid': float(raw_score_result['valid']),
+                'error_message': raw_score_result.get('error_message', ''),
+                'error_type': raw_score_result.get('error_type', ''),
+            }
+        else:
+            try:
+                if self.is_fc_model:
+                    decoded_tool_calls = []
+                    for tool_call in row['generation'][0]:
+                        name = list(tool_call.keys())[0]
+                        params = json.loads(tool_call[name])
+                        decoded_tool_calls.append({name: params})
+                else:
+                    decoded_tool_calls = default_decode_ast_prompting(row['generation'][0][0], row['language'])
+
+                score_result = ast_checker(
+                    row['functions'],
+                    decoded_tool_calls,
+                    row['ground_truth'],
+                    row['language'],
+                    row['test_category'],
+                    dummy_model,
+                )
+            except Exception:
+                score_result = {
+                    'valid': False,
+                    'error_message': f'Invalid syntax. Failed to decode AST. Traceback: {traceback.format_exc()}',
+                    'error_type': 'ast_decoder:decoder_failed',
+                }
+
+        return {
+            'AverageAccuracy': float(score_result['valid']),
+            'raw_score_result': score_result,
+        }
+
+    def compute_metric(self, review_res_list: List[dict], **kwargs) -> Any:
+        # aggregate review results
+        res_dict = super().compute_dict_metric(review_res_list, **kwargs)
+
+        return super().compute_metric(res_dict, **kwargs)

evalscope/benchmarks/ceval/ceval_adapter.py

@@ -126,6 +126,9 @@ SUBJECT_MAPPING = {
 @Benchmark.register(
     name='ceval',
     pretty_name='C-Eval',
+    tags=['Knowledge', 'MCQ', 'Chinese'],
+    description=
+    'C-Eval is a benchmark designed to evaluate the performance of AI models on Chinese exams across various subjects, including STEM, social sciences, and humanities. It consists of multiple-choice questions that test knowledge and reasoning abilities in these areas.',  # noqa: E501
     dataset_id='modelscope/ceval-exam',
     model_adapter=OutputType.GENERATION,
     output_types=[OutputType.MULTIPLE_CHOICE, OutputType.GENERATION],

evalscope/benchmarks/chinese_simple_qa/csimple_qa_adapter.py

@@ -87,7 +87,10 @@ SUBSET_LIST = ['中华文化', '人文与社会科学', '工程、技术与应
 
 @Benchmark.register(
     name='chinese_simpleqa',
-    pretty_name='Chinese SimpleQA',
+    pretty_name='Chinese-SimpleQA',
+    tags=['Knowledge', 'QA', 'Chinese'],
+    description=
+    "Chinese SimpleQA is a Chinese question-answering dataset designed to evaluate the performance of language models on simple factual questions. It includes a variety of topics and is structured to test the model's ability to understand and generate correct answers in Chinese.",  # noqa: E501
     subset_list=SUBSET_LIST,
     dataset_id='AI-ModelScope/Chinese-SimpleQA',
     metric_list=['is_correct', 'is_incorrect', 'is_not_attempted'],

evalscope/benchmarks/cmmlu/cmmlu_adapter.py

@@ -103,6 +103,9 @@ SUBJECT_MAPPING = {
 @Benchmark.register(
     name='cmmlu',
     pretty_name='C-MMLU',
+    tags=['Knowledge', 'MCQ', 'Chinese'],
+    description=
+    'C-MMLU is a benchmark designed to evaluate the performance of AI models on Chinese language tasks, including reading comprehension, text classification, and more.',
     dataset_id='modelscope/cmmlu',
     model_adapter=OutputType.GENERATION,
     output_types=[OutputType.MULTIPLE_CHOICE, OutputType.GENERATION],

evalscope/benchmarks/competition_math/competition_math_adapter.py

@@ -17,6 +17,9 @@ logger = get_logger()
 @Benchmark.register(
     name='competition_math',
     pretty_name='MATH',
+    tags=['Mathematics'],
+    description=
+    'The MATH (Mathematics) benchmark is designed to evaluate the mathematical reasoning abilities of AI models through a variety of problem types, including arithmetic, algebra, geometry, and more.',
     dataset_id='modelscope/competition_math',
     subset_list=['Level 1', 'Level 2', 'Level 3', 'Level 4', 'Level 5'],
     metric_list=['AveragePass@1'],

evalscope/benchmarks/data_adapter.py

@@ -5,7 +5,7 @@ from abc import ABC, abstractmethod
 from collections import defaultdict
 from typing import Any, Dict, List, Optional, Union
 
-from evalscope.benchmarks.utils import PromptData, preprocess_decorator
+from evalscope.benchmarks.utils import PromptData, load_file_with_extension, preprocess_decorator
 from evalscope.constants import DEFAULT_DATASET_CACHE_DIR, AnswerKeys, EvalType, HubType
 from evalscope.metrics import LLMJudge, metric_registry
 from evalscope.report import Report, ReportGenerator
@@ -15,6 +15,13 @@ logger = get_logger()
 
 
 class DataAdapter(ABC):
+    """
+    Data Adapter for the benchmark. You need to implement the following methods:
+        - gen_prompt
+        - get_gold_answer
+        - parse_pred_result
+        - match
+    """
 
     def __init__(self,
                  name: str,
@@ -31,30 +38,37 @@ class DataAdapter(ABC):
                  system_prompt: Optional[str] = None,
                  query_template: Optional[str] = None,
                  pretty_name: Optional[str] = None,
+                 description: Optional[str] = None,
+                 tags: Optional[List[str]] = None,
                  **kwargs):
         """
-        Data Adapter for the benchmark. You need to implement the following methods:
-            - gen_prompt
-            - get_gold_answer
-            - parse_pred_result
-            - match
         Args:
             name: str, the name of the benchmark.
             dataset_id: str, the dataset id on ModelScope or local path for the benchmark.
+            model_adapter: str, the model adapter to use for the benchmark.
             subset_list: list of subset names for the dataset.
             metric_list: list, the metric list to evaluate the model on specific benchmark.
+            llm_as_a_judge: bool, whether to use LLM as a judge to evaluate the predicted answer against the gold answer.
+            output_types: list, the output types of the model adapter. Default: [model_adapter]
             few_shot_num: int, number of few-shot examples. Default: 0
             train_split: str, usually for few-shot examples. e.g. 'train'
             eval_split: str, the target eval split name. e.g. 'test'
             prompt_template: str, the prompt template for the benchmark,
                 e.g. for ARC, it is `The following are multiple choice questions, please output correct answer in
                     the form of A or B or C or D, do not output explanation:`
-        """
+            system_prompt: str, the system prompt for the benchmark, e.g. 'You are a helpful assistant.'
+            query_template: str, the query template for the benchmark, e.g. 'Please answer the following question: {}'
+            pretty_name: str, the pretty name of the benchmark, e.g. 'ARC Challenge Set'.
+            description: str, the description of the benchmark,
+                e.g. 'ARC Challenge Set is a benchmark for evaluating reasoning abilities of models on science questions.'
+        """  # noqa: E501
         self.name = name
         self.dataset_id = dataset_id
         self.model_adapter = model_adapter
         self.subset_list = subset_list
         self.metric_list = metric_list
+        self.llm_as_a_judge = llm_as_a_judge
+        self.output_types = output_types or [model_adapter]
         self.few_shot_num = few_shot_num
         self.train_split = train_split
         self.eval_split = eval_split
@@ -62,9 +76,9 @@ class DataAdapter(ABC):
         self.system_prompt = system_prompt
         self.query_template = query_template
         self.pretty_name = pretty_name
+        self.description = description
+        self.tags = tags or []
         self.config_kwargs = kwargs
-        self.output_types = output_types or [model_adapter]
-        self.llm_as_a_judge = llm_as_a_judge
         self.category_map = kwargs.get('category_map', {})
         self.choices = kwargs.get('choices', None)
 
@@ -156,6 +170,49 @@ class DataAdapter(ABC):
         """
         return self.load_from_hub(dataset_name_or_path, subset_list, work_dir, **kwargs)
 
+    def load_with_snapshot(self,
+                           file_structure: Dict[str, List[str]],
+                           dataset_name_or_path: str = None,
+                           subset_list: list = None,
+                           work_dir: Optional[str] = DEFAULT_DATASET_CACHE_DIR,
+                           **kwargs) -> dict:
+        """
+        For datasets that cannot be correctly loaded using MsDataset, utilize snapshot downloading to load the data.
+        This feature supports both remote and local datasets.
+
+        Args:
+            file_structure: dict, the file structure of the dataset, e.g. {'subset_name': ['file1.jsonl', 'file2.jsonl']}.
+            dataset_name_or_path: str, the dataset id on ModelScope or local path for the benchmark.
+            subset_list: list of subset names for the dataset.
+            work_dir: str, the working directory to store the dataset.
+        Returns: {'subset_name': {'eval': eval_dataset}}
+        """  # noqa: E501
+        dataset_name_or_path = os.path.expanduser(dataset_name_or_path or self.dataset_id)
+        subset_list = subset_list or self.subset_list
+
+        # Try to load dataset from local disk
+        if os.path.exists(dataset_name_or_path):
+            logger.info(f'Loading dataset from {dataset_name_or_path}')
+            dataset_path = dataset_name_or_path
+        else:
+            from modelscope import dataset_snapshot_download
+
+            # Load dataset from remote
+            logger.info(f'Loading dataset from modelscope: > dataset_name: {dataset_name_or_path}')
+            # flatten file structure
+            file_names = [file for sub_files in file_structure.values() for file in sub_files]
+            # download dataset snapshot
+            dataset_path = dataset_snapshot_download(
+                dataset_name_or_path, cache_dir=work_dir, allow_file_pattern=file_names)
+        # read and process files
+        data_dict = defaultdict(dict)
+        for sub_name in subset_list:
+            file_paths = [os.path.join(dataset_path, file_name) for file_name in file_structure[sub_name]]
+            # not train split, only eval split
+            data_dict[sub_name][self.eval_split] = load_file_with_extension(file_paths)
+
+        return data_dict
+
     def reformat_subset(self, data_dict: dict, subset_key: str, format: str = '{}') -> dict:
         """
         Reformat the dataset subset with subset_key and format.
@@ -249,7 +306,7 @@ class DataAdapter(ABC):
     def compute_dict_metric(self, review_res_list: Union[List[dict], List[List[dict]]],
                             **kwargs) -> Dict[str, List[float]]:
         """
-        compute weighted mean of the bleu score of all samples
+        compute weighted mean of score of all samples
 
         Args:
             review_res_list: [score1, score2, ...]
@@ -270,7 +327,7 @@ class DataAdapter(ABC):
                 items['AverageAccuracy'].append(scores)
         return items
 
-    def gen_report(self, subset_score_map: dict, report_name: str = None, **kwargs) -> Report:
+    def gen_report(self, subset_score_map: dict, model_name: str, **kwargs) -> Report:
         """
         Generate report for the evaluation results for all subsets.
 
@@ -278,7 +335,7 @@ class DataAdapter(ABC):
             subset_score_map: The subset-score map.
                 e.g. {subset_name: [{'metric_name': 'AverageAccuracy', 'score': 0.3389, 'num': 100}]}
 
-            report_name: str, the user-defined report name. Default: None
+            model_name: The evaluation model name.
 
         Returns: The evaluation report.
 
@@ -312,9 +369,17 @@ class DataAdapter(ABC):
             "model_name": "qwen2.5"
         }
         """  # noqa: E501
-        kwargs['category_map'] = self.category_map
-        kwargs['metric_list'] = self.metric_list
-        return ReportGenerator.gen_report(subset_score_map, report_name, **kwargs)
+        return ReportGenerator.gen_report(subset_score_map, model_name, data_adapter=self, **kwargs)
+
+    def post_process_report(self, report: Report, **kwargs):
+        """
+        Post-process the report after generation. Draw a chart, save to file, etc.
+        This method can be overridden to customize the report format or content.
+
+        Args:
+            report (Report): The generated report.
+        """
+        pass
 
     def gen_prompt_data(self,
                         prompt: str,
@@ -324,6 +389,23 @@ class DataAdapter(ABC):
                         id: Optional[Union[int, str]] = None,
                         messages: Optional[List[dict]] = None,
                         **kwargs) -> dict:
+        """
+        Generates a dictionary representation of prompt data for evaluation or inference.
+
+        Args:
+            prompt (str): The main prompt or input text. Can also be a list of prompts.
+            system_prompt (Optional[str], optional): An optional system-level prompt to provide context or instructions. Defaults to None.
+            choices (Optional[List[str]], optional): A list of possible choices for multi-choice tasks.
+                If not provided, uses self.choices. Defaults to None.
+            index (Optional[Union[int, str]], optional): An optional index or identifier for the prompt.
+                Defaults to 0 if not provided. Defaults to None.
+            id (Optional[Union[int, str]], optional): An optional unique identifier for the prompt data. Defaults to None.
+            messages (Optional[List[dict]], optional): An optional list of message dictionaries, typically for chat-based prompts. Defaults to None.
+                If messages is provided, it will be used as the prompt data instead of the prompt string.
+
+        Returns:
+            dict: A dictionary representation of the prompt data, suitable for further processing or model input.
+        """  # noqa: E501
         data = [prompt] if not isinstance(prompt, list) else prompt
         prompt_data = PromptData(
             data=data,
@@ -416,7 +498,8 @@ class DataAdapter(ABC):
 
         # Extract question from raw_input if available
         raw_input = kwargs.get('raw_input', {})
-        question_keys = ['question', 'prompt', 'query', 'problem']
+        question_keys = ['question', 'Question', 'prompt', 'Prompt', 'query', 'Query', 'problem', 'Problem']
+        # Find the first non-empty question key in raw_input
         question = next((raw_input.get(key) for key in question_keys if raw_input.get(key)), None)
 
         # Request judge and obtain score

evalscope/benchmarks/data_collection/data_collection_adapter.py

@@ -14,6 +14,7 @@ logger = get_logger()
 @Benchmark.register(
     name='data_collection',
     dataset_id='',  # dataset_id need to be set
+    description='Data collection',
     subset_list=['default'],
     metric_list=['AverageAccuracy'],
     few_shot_num=0,