evalscope 0.16.1__py3-none-any.whl → 0.16.3__py3-none-any.whl

evalscope/app/app.py

@@ -1,6 +1,7 @@
 import argparse
 import glob
 import gradio as gr
+import json
 import numpy as np
 import os
 import pandas as pd
@@ -135,11 +136,11 @@ def plot_single_report_scores(df: pd.DataFrame):
 
 def plot_single_report_sunburst(report_list: List[Report]):
     if report_list[0].name == DataCollection.NAME:
-        df = get_data_frame(report_list)
+        df = get_data_frame(report_list=report_list)
         categories = sorted([i for i in df.columns if i.startswith(ReportKey.category_prefix)])
         path = categories + [ReportKey.subset_name]
     else:
-        df = get_data_frame(report_list, flatten_metrics=False)
+        df = get_data_frame(report_list=report_list, flatten_metrics=False)
         categories = sorted([i for i in df.columns if i.startswith(ReportKey.category_prefix)])
         path = [ReportKey.dataset_name] + categories + [ReportKey.subset_name]
     logger.debug(f'df: {df}')
@@ -233,7 +234,7 @@ def convert_html_tags(text):
 def process_string(string: str, max_length: int = 2048) -> str:
     string = convert_html_tags(string)  # for display labels e.g.
     if max_length and len(string) > max_length:
-        return f'{string[:max_length // 2]}......{string[-max_length // 2:]}'
+        return f'{string[:max_length // 2]}...[truncate]...{string[-max_length // 2:]}'
     return string
 
 
@@ -257,7 +258,7 @@ def dict_to_markdown(data) -> str:
     return '\n\n'.join(markdown_lines)
 
 
-def process_model_prediction(item: Any, max_length: int = 2048) -> str:
+def process_model_prediction_old(item: Any, max_length: int = 2048) -> str:
     """
     Process model prediction output into a formatted string.
 
@@ -281,6 +282,20 @@ def process_model_prediction(item: Any, max_length: int = 2048) -> str:
     return result
 
 
+def process_model_prediction(item: Any, max_length: int = 4096) -> str:
+    if isinstance(item, (dict, list)):
+        result = json.dumps(item, ensure_ascii=False, indent=2)
+        result = f'```json\n{result}\n```'
+    else:
+        result = str(item)
+
+    # Apply HTML tag conversion and truncation only at the final output
+    if max_length is not None:
+        return process_string(result, max_length)
+
+    return result
+
+
 def normalize_score(score):
     try:
         if isinstance(score, bool):
@@ -583,7 +598,7 @@ def create_single_model_tab(sidebar: SidebarComponents, lang: str):
         outputs=[dataset_plot, dataset_table, subset_select, data_review_df, report_analysis])
     def update_single_report_dataset(dataset_name, report_list):
         logger.debug(f'Updating single report dataset: {dataset_name}')
-        report_df = get_data_frame(report_list)
+        report_df = get_data_frame(report_list=report_list)
         analysis = get_report_analysis(report_list, dataset_name)
         data_score_df, styler = get_single_dataset_df(report_df, dataset_name)
         data_score_plot = plot_single_dataset_scores(data_score_df)

evalscope/backend/rag_eval/clip_benchmark/dataset_builder.py

@@ -1,4 +1,5 @@
 import os
+import posixpath  # For URL path handling
 import torch
 from torch.utils.data import DataLoader
 from torch.utils.data import Dataset as TorchDataset
@@ -186,42 +187,53 @@ def build_wds_dataset(dataset_name, transform, split='test', data_dir='root', ca
 
     Set `cache_dir` to a path to cache the dataset, otherwise, no caching will occur.
     """
+    import requests
     import webdataset as wds
 
     def read_txt(fname):
-        if '://' in fname:
-            stream = os.popen("curl -L -s --fail '%s'" % fname, 'r')
-            value = stream.read()
-            if stream.close():
-                raise FileNotFoundError('Failed to retreive data')
+        if fname.startswith(('http://', 'https://')):
+            try:
+                response = requests.get(fname)
+                response.raise_for_status()  # Ensure the HTTP request was successful
+                return response.text
+            except requests.exceptions.RequestException as e:
+                raise FileNotFoundError(f'Failed to read {fname}: {e}')
         else:
             with open(fname, 'r') as file:
-                value = file.read()
-        return value
+                return file.read()
+
+    def url_path_join(*parts):
+        """Join URL path parts with forward slashes regardless of platform"""
+        return posixpath.join(*parts)
 
     if not data_dir:
         data_dir = f'https://modelscope.cn/datasets/clip-benchmark/wds_{dataset_name}/resolve/master'
 
     # Git LFS files have a different file path to access the raw data than other files
-    if data_dir.startswith('https://modelscope.cn/datasets'):
+    is_url = data_dir.startswith(('http://', 'https://'))
+    if is_url and data_dir.startswith('https://modelscope.cn/datasets'):
         *split_url_head, _, url_path = data_dir.split('/', 7)
         url_head = '/'.join(split_url_head)
         metadata_dir = '/'.join([url_head, 'resolve', url_path])
         tardata_dir = '/'.join([url_head, 'resolve', url_path])
     else:
         metadata_dir = tardata_dir = data_dir
+
+    # Use appropriate path joining function based on whether we're dealing with a URL
+    path_join = url_path_join if is_url else os.path.join
+
     # Get number of shards
-    nshards_fname = os.path.join(metadata_dir, split, 'nshards.txt')
+    nshards_fname = path_join(metadata_dir, split, 'nshards.txt')
     nshards = int(read_txt(nshards_fname))  # Do not catch FileNotFound, nshards.txt should be mandatory
 
     # Get dataset type (classification or retrieval)
-    type_fname = os.path.join(metadata_dir, 'dataset_type.txt')
+    type_fname = path_join(metadata_dir, 'dataset_type.txt')
     try:
         dataset_type = read_txt(type_fname).strip().lower()
     except FileNotFoundError:
         dataset_type = 'classification'
 
-    filepattern = os.path.join(tardata_dir, split, '{0..%d}.tar' % (nshards - 1))
+    filepattern = path_join(tardata_dir, split, '{0..%d}.tar' % (nshards - 1))
     # Load webdataset (support WEBP, PNG, and JPG for now)
     if not cache_dir or not isinstance(cache_dir, str):
         cache_dir = None

evalscope/backend/rag_eval/utils/embedding.py

@@ -172,11 +172,9 @@ class CrossEncoderModel(BaseModel):
                 kwargs.pop(key)
         self.encode_kwargs.update(kwargs)
 
-        if len(sentences[0]) == 3:  # Note: For mteb retrieval task
+        if len(sentences[0]) == 2:  # Note: For mteb retrieval task
             processed_sentences = []
-            for query, docs, instruction in sentences:
-                if isinstance(docs, dict):
-                    docs = docs['text']
+            for query, docs in sentences:
                 processed_sentences.append((self.prompt + query, docs))
             sentences = processed_sentences
         embeddings = self.model.predict(sentences, **self.encode_kwargs)

evalscope/benchmarks/aigc/t2i/evalmuse_adapter.py

@@ -69,6 +69,7 @@ class EvalMuseAdapter(T2IBaseAdapter):
             if 'FGA_BLIP2Score' in metric_name and '(' in metric_name:  # FGA_BLIP2Score element score
                 metrics_prefix = metric_name.split(':')[0]
                 category = metric_name.rpartition('(')[-1].split(')')[0]
+                category = category.split('-')[0].lower()  # remove the suffix if exists
                 new_items[f'{metrics_prefix}:{category}'].extend(value_list)
             else:
                 new_items[metric_name].extend(value_list)

evalscope/benchmarks/aime/aime24_adapter.py

@@ -1,5 +1,4 @@
 from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.constants import OutputType
 from evalscope.metrics import extract_answer, math_equal, strip_answer_string
 from evalscope.utils.logger import get_logger
 
@@ -11,6 +10,9 @@ logger = get_logger()
 @Benchmark.register(
     name='aime24',
     pretty_name='AIME-2024',
+    tags=['Mathematics'],
+    description=
+    'The AIME 2024 benchmark is based on problems from the American Invitational Mathematics Examination, a prestigious high school mathematics competition. This benchmark tests a model’s ability to solve challenging mathematics problems by generating step-by-step solutions and providing the correct final answer.',  # noqa: E501
     dataset_id='HuggingFaceH4/aime_2024',
     subset_list=['default'],
     metric_list=['AveragePass@1'],

evalscope/benchmarks/aime/aime25_adapter.py

@@ -1,5 +1,4 @@
 from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.constants import OutputType
 from evalscope.metrics import extract_answer, math_equal, strip_answer_string
 from evalscope.utils.logger import get_logger
 
@@ -11,6 +10,9 @@ logger = get_logger()
 @Benchmark.register(
     name='aime25',
     pretty_name='AIME-2025',
+    tags=['Mathematics'],
+    description=
+    'The AIME 2025 benchmark is based on problems from the American Invitational Mathematics Examination, a prestigious high school mathematics competition. This benchmark tests a model’s ability to solve challenging mathematics problems by generating step-by-step solutions and providing the correct final answer.',
     dataset_id='opencompass/AIME2025',
     subset_list=['AIME2025-I', 'AIME2025-II'],
     metric_list=['AveragePass@1'],

evalscope/benchmarks/alpaca_eval/alpaca_eval_adapter.py

@@ -47,6 +47,11 @@ Evaluate the models based on the quality and relevance of their outputs, and sel
 @Benchmark.register(
     name='alpaca_eval',
     pretty_name='AlpacaEval2.0',
+    tags=['Instruction-Following', 'Reasoning'],
+    description='Alpaca Eval 2.0 is an enhanced framework for evaluating instruction-following language models, '
+    'featuring an improved auto-annotator, updated baselines, and continuous preference calculation to '
+    'provide more accurate and cost-effective model assessments. '
+    'Currently not support `length-controlled winrate`; the official Judge model is `gpt-4-1106-preview`, while the baseline model is `gpt-4-turbo`.',  # noqa: E501
     dataset_id='AI-ModelScope/alpaca_eval',
     subset_list=['alpaca_eval_gpt4_baseline'],
     metric_list=['winrate'],

evalscope/benchmarks/arc/arc_adapter.py

@@ -17,6 +17,9 @@ logger = get_logger()
 @Benchmark.register(
     name='arc',
     pretty_name='ARC',
+    tags=['Reasoning', 'MCQ'],
+    description=
+    'The ARC (AI2 Reasoning Challenge) benchmark is designed to evaluate the reasoning capabilities of AI models through multiple-choice questions derived from science exams. It includes two subsets: ARC-Easy and ARC-Challenge, which vary in difficulty.',  # noqa: E501
     dataset_id='modelscope/ai2_arc',
     model_adapter=OutputType.GENERATION,
     output_types=[OutputType.MULTIPLE_CHOICE, OutputType.GENERATION],

evalscope/benchmarks/arena_hard/arena_hard_adapter.py

@@ -1,5 +1,3 @@
-import re
-from collections import defaultdict
 from typing import Any, List
 
 from evalscope.benchmarks import Benchmark, DataAdapter
@@ -19,12 +17,18 @@ GRADER_TEMPLATE = "<|User Prompt|>\n{question}\n\n<|The Start of Assistant A's A
 @Benchmark.register(
     name='arena_hard',
     pretty_name='ArenaHard',
+    tags=['Instruction-Following', 'Reasoning'],
+    description=
+    'ArenaHard is a benchmark designed to evaluate the performance of large language models in a competitive setting, '
+    'where models are pitted against each other in a series of tasks to determine their relative strengths and weaknesses. '
+    'It includes a set of challenging tasks that require reasoning, understanding, and generation capabilities. '
+    'Currently not support `style-controlled winrate`; the official Judge model is `gpt-4-1106-preview`, while the baseline model is `gpt-4-0314`.',  # noqa: E501
     dataset_id='AI-ModelScope/arena-hard-auto-v0.1',
     metric_list=['winrate'],
     few_shot_num=0,
     train_split=None,
     eval_split='test')
-class AlpacaEvalAdapter(DataAdapter):
+class ArenaHardAdapter(DataAdapter):
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)

evalscope/benchmarks/bbh/bbh_adapter.py

@@ -59,6 +59,9 @@ SUBSET_LIST = MULTIPLE_CHOICE_LIST + FREE_FORM_LIST
 @Benchmark.register(
     name='bbh',
     pretty_name='BBH',
+    tags=['Reasoning'],
+    description=
+    'The BBH (Big Bench Hard) benchmark is a collection of challenging tasks designed to evaluate the reasoning capabilities of AI models. It includes both free-form and multiple-choice tasks, covering a wide range of reasoning skills.',  # noqa: E501
     dataset_id='modelscope/bbh',
     subset_list=SUBSET_LIST,
     metric_list=['AverageAccuracy'],

evalscope/benchmarks/benchmark.py

@@ -29,6 +29,7 @@ class BenchmarkMeta:
     query_template: Optional[str] = None
     pretty_name: Optional[str] = None
     description: Optional[str] = None
+    tags: Optional[List[str]] = field(default_factory=list)
     filters: Optional[OrderedDict] = None
     extra_params: Optional[Dict] = field(default_factory=dict)
 

evalscope/benchmarks/bfcl/bfcl_adapter.py

@@ -0,0 +1,237 @@
+import copy
+import importlib
+import json
+import re
+import traceback
+from typing import Any, List
+
+from evalscope.benchmarks import Benchmark, DataAdapter
+from evalscope.constants import EvalType
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+SUBJECT_MAPPING = {
+    'simple': 'AST_NON_LIVE',
+    'multiple': 'AST_NON_LIVE',
+    'parallel': 'AST_NON_LIVE',
+    'parallel_multiple': 'AST_NON_LIVE',
+    'java': 'AST_NON_LIVE',
+    'javascript': 'AST_NON_LIVE',
+    'live_simple': 'AST_LIVE',
+    'live_multiple': 'AST_LIVE',
+    'live_parallel': 'AST_LIVE',
+    'live_parallel_multiple': 'AST_LIVE',
+    'irrelevance': 'RELEVANCE',
+    'live_relevance': 'RELEVANCE',
+    'live_irrelevance': 'RELEVANCE',
+    'multi_turn_base': 'MULTI_TURN',
+    'multi_turn_miss_func': 'MULTI_TURN',
+    'multi_turn_miss_param': 'MULTI_TURN',
+    'multi_turn_long_context': 'MULTI_TURN'
+}
+
+
+@Benchmark.register(
+    name='bfcl_v3',
+    pretty_name='BFCL-v3',
+    tags=['Agent'],
+    description=
+    'Berkeley Function Calling Leaderboard (BFCL), the **first comprehensive and executable function call evaluation** '
+    'dedicated to assessing Large Language Models\' (LLMs) ability to invoke functions. Unlike previous evaluations, '
+    'BFCL accounts for various forms of function calls, diverse scenarios, and executability. '
+    'Need to run `pip install bfcl-eval` before evaluating. '
+    '[Usage Example](https://evalscope.readthedocs.io/zh-cn/latest/third_party/bfcl_v3.html)',  # noqa: E501
+    dataset_id='AI-ModelScope/bfcl_v3',
+    subset_list=list(SUBJECT_MAPPING.keys()),
+    model_adapter='bfcl_server',
+    metric_list=['AverageAccuracy'],
+    few_shot_num=0,
+    train_split=None,
+    eval_split='train',
+    extra_params={
+        'underscore_to_dot': True,
+        'is_fc_model': True,
+    })
+class BFCLAdapter(DataAdapter):
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+        spec = importlib.util.find_spec('bfcl_eval')
+        if spec is None:
+            raise ImportError(
+                '`bfcl_eval` not found, please install it with `pip install bfcl-eval` before evaluating.')
+
+        self.category_map = SUBJECT_MAPPING
+
+        extra_params = kwargs.get('extra_params', {})
+        self.underscore_to_dot = extra_params.get('underscore_to_dot', False)
+        self.is_fc_model = extra_params.get('is_fc_model', True)
+
+    def load(self, **kwargs):
+        kwargs['subset_list'] = ['default']
+        data_dict = super().load(**kwargs)
+        return self.reformat_subset(data_dict, subset_key='subset', format='{}')
+
+    def preprocess_row(self, row: dict):
+        """
+        Inplace preprocess the row to ensure it has the correct format for BFCL evaluation.
+        """
+        row['should_execute_tool_calls'] = True if row['multi_turn'] else False
+        row['functions'] = json.loads(row['functions'])
+        row['tools'] = json.loads(row['tools'])
+        row['turns'] = json.loads(row['turns'])
+        row['missing_functions'] = json.loads(row['missed_functions'])
+        row['ground_truth'] = json.loads(row.get('ground_truth', '{}'))
+        row['initial_config'] = json.loads(row['initial_config'])
+        row['is_fc_model'] = self.is_fc_model
+
+    def gen_prompt(self, input_d, subset_name, few_shot_list, **kwargs):
+        self.preprocess_row(input_d)
+
+        # If the model is a function calling model, we need to remove the system prompt
+        if self.is_fc_model:
+            turns = input_d['turns']
+            new_turns = []
+            for turn_idx, messages in enumerate(turns):
+                current_messages = messages.copy()
+                if len(current_messages) > 0 and current_messages[0]['role'] == 'system':
+                    current_messages = current_messages[1:]
+                new_turns.append(current_messages)
+            input_d['turns'] = new_turns
+
+        return self.gen_prompt_data(prompt='', messages=input_d)
+
+    def get_gold_answer(self, input_d: dict) -> str:
+        # Get the gold choice
+        return input_d.get('ground_truth', )
+
+    def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = EvalType.CHECKPOINT) -> dict:
+        row = copy.deepcopy(raw_input_d)
+        del row['turns']  # Remove turns as they are not needed for the match function
+
+        row['generation'] = result
+        return row
+
+    def match(self, gold: dict, pred: dict) -> dict:
+        from bfcl_eval.eval_checker.ast_eval.ast_checker import ast_checker
+        from bfcl_eval.eval_checker.multi_turn_eval.multi_turn_checker import multi_turn_checker
+        from bfcl_eval.model_handler.utils import (convert_to_function_call, default_decode_ast_prompting,
+                                                   default_decode_execute_prompting)
+        from bfcl_eval.utils import is_empty_output
+
+        # NOTE: This is hardcoded dummy model since its only use is to infer underscore_to_dot
+        # which decides if model was provided with functions of the type
+        # spotify.list_songs or spotify_list_songs
+        # It is False for all llama models (when using via prompting)
+        # and True for API calls
+        if self.underscore_to_dot:
+            dummy_model = 'gpt-4o-2024-11-20-FC'
+        else:
+            dummy_model = 'meta-llama/Llama-3.3-70B-Instruct-FC'
+
+        row = pred
+        test_category = re.sub(r'_[0-9_-]+$', '', row['id'])
+        if test_category in {'irrelevance', 'live_irrelevance', 'live_relevance'}:
+            error = None
+            try:
+                if self.is_fc_model:
+                    decoded_tool_calls = []
+                    for tool_call in row['generation'][0]:
+                        name = list(tool_call.keys())[0]
+                        params = json.loads(tool_call[name])
+                        decoded_tool_calls.append({name: params})
+                else:
+                    decoded_tool_calls = default_decode_ast_prompting(row['generation'][0][0], row['language'])
+
+                # successful decode means valid function call was present
+                contains_func_call = True
+                if is_empty_output(decoded_tool_calls):
+                    # Empty output is not considered as a valid function call
+                    contains_func_call = False
+                    error = 'Empty decoded output.'
+            except Exception:
+                contains_func_call = False
+                error = f'Failed to decode with traceback: {traceback.format_exc()}'
+            finally:
+                valid = contains_func_call if test_category == 'live_relevance' else not contains_func_call
+                score_result = {'valid': valid, 'error_message': error}
+
+        elif row['multi_turn']:
+            # each step might give a list of tool calls and each turn is multi-step
+            # and multi-turn has generations of all the turns
+            # hence in a multi-turn setting,
+            # multi_turn_decoded_generations is a list of list of list of strings
+            multi_turn_decoded_generations: list[list[list[str]]] = []
+            for single_turn_generations in row['generation']:
+                single_turn_decoded_generations: list[list[str]] = []
+                for generation in single_turn_generations:
+                    try:
+                        if self.is_fc_model:
+                            tool_calls = convert_to_function_call(generation)
+                        else:
+                            tool_calls = default_decode_execute_prompting(generation)
+
+                        single_turn_decoded_generations.append(tool_calls)
+                    except Exception:
+                        single_turn_decoded_generations.append([generation])
+
+                multi_turn_decoded_generations.append(single_turn_decoded_generations)
+
+            try:
+                raw_score_result = multi_turn_checker(
+                    multi_turn_decoded_generations,
+                    row['ground_truth'],
+                    row,
+                    test_category,
+                    dummy_model,
+                )
+            except Exception:
+                raw_score_result = {
+                    'valid': False,
+                    'error_type': 'multi_turn:checker_failed',
+                    'error_message': f'Failed to grade multi-turn. Traceback: {traceback.format_exc()}',
+                }
+
+            score_result = {
+                'valid': float(raw_score_result['valid']),
+                'error_message': raw_score_result.get('error_message', ''),
+                'error_type': raw_score_result.get('error_type', ''),
+            }
+        else:
+            try:
+                if self.is_fc_model:
+                    decoded_tool_calls = []
+                    for tool_call in row['generation'][0]:
+                        name = list(tool_call.keys())[0]
+                        params = json.loads(tool_call[name])
+                        decoded_tool_calls.append({name: params})
+                else:
+                    decoded_tool_calls = default_decode_ast_prompting(row['generation'][0][0], row['language'])
+
+                score_result = ast_checker(
+                    row['functions'],
+                    decoded_tool_calls,
+                    row['ground_truth'],
+                    row['language'],
+                    row['test_category'],
+                    dummy_model,
+                )
+            except Exception:
+                score_result = {
+                    'valid': False,
+                    'error_message': f'Invalid syntax. Failed to decode AST. Traceback: {traceback.format_exc()}',
+                    'error_type': 'ast_decoder:decoder_failed',
+                }
+
+        return {
+            'AverageAccuracy': float(score_result['valid']),
+            'raw_score_result': score_result,
+        }
+
+    def compute_metric(self, review_res_list: List[dict], **kwargs) -> Any:
+        # aggregate review results
+        res_dict = super().compute_dict_metric(review_res_list, **kwargs)
+
+        return super().compute_metric(res_dict, **kwargs)

evalscope/benchmarks/ceval/ceval_adapter.py

@@ -126,6 +126,9 @@ SUBJECT_MAPPING = {
 @Benchmark.register(
     name='ceval',
     pretty_name='C-Eval',
+    tags=['Knowledge', 'MCQ', 'Chinese'],
+    description=
+    'C-Eval is a benchmark designed to evaluate the performance of AI models on Chinese exams across various subjects, including STEM, social sciences, and humanities. It consists of multiple-choice questions that test knowledge and reasoning abilities in these areas.',  # noqa: E501
     dataset_id='modelscope/ceval-exam',
     model_adapter=OutputType.GENERATION,
     output_types=[OutputType.MULTIPLE_CHOICE, OutputType.GENERATION],

evalscope/benchmarks/chinese_simple_qa/csimple_qa_adapter.py

@@ -87,7 +87,10 @@ SUBSET_LIST = ['中华文化', '人文与社会科学', '工程、技术与应
 
 @Benchmark.register(
     name='chinese_simpleqa',
-    pretty_name='Chinese SimpleQA',
+    pretty_name='Chinese-SimpleQA',
+    tags=['Knowledge', 'QA', 'Chinese'],
+    description=
+    "Chinese SimpleQA is a Chinese question-answering dataset designed to evaluate the performance of language models on simple factual questions. It includes a variety of topics and is structured to test the model's ability to understand and generate correct answers in Chinese.",  # noqa: E501
     subset_list=SUBSET_LIST,
     dataset_id='AI-ModelScope/Chinese-SimpleQA',
     metric_list=['is_correct', 'is_incorrect', 'is_not_attempted'],

evalscope/benchmarks/cmmlu/cmmlu_adapter.py

@@ -103,6 +103,9 @@ SUBJECT_MAPPING = {
 @Benchmark.register(
     name='cmmlu',
     pretty_name='C-MMLU',
+    tags=['Knowledge', 'MCQ', 'Chinese'],
+    description=
+    'C-MMLU is a benchmark designed to evaluate the performance of AI models on Chinese language tasks, including reading comprehension, text classification, and more.',
     dataset_id='modelscope/cmmlu',
     model_adapter=OutputType.GENERATION,
     output_types=[OutputType.MULTIPLE_CHOICE, OutputType.GENERATION],

evalscope/benchmarks/competition_math/competition_math_adapter.py

@@ -17,6 +17,9 @@ logger = get_logger()
 @Benchmark.register(
     name='competition_math',
     pretty_name='MATH',
+    tags=['Mathematics'],
+    description=
+    'The MATH (Mathematics) benchmark is designed to evaluate the mathematical reasoning abilities of AI models through a variety of problem types, including arithmetic, algebra, geometry, and more.',
     dataset_id='modelscope/competition_math',
     subset_list=['Level 1', 'Level 2', 'Level 3', 'Level 4', 'Level 5'],
     metric_list=['AveragePass@1'],

evalscope/benchmarks/data_adapter.py

@@ -39,6 +39,7 @@ class DataAdapter(ABC):
                  query_template: Optional[str] = None,
                  pretty_name: Optional[str] = None,
                  description: Optional[str] = None,
+                 tags: Optional[List[str]] = None,
                  **kwargs):
         """
         Args:
@@ -76,6 +77,7 @@ class DataAdapter(ABC):
         self.query_template = query_template
         self.pretty_name = pretty_name
         self.description = description
+        self.tags = tags or []
         self.config_kwargs = kwargs
         self.category_map = kwargs.get('category_map', {})
         self.choices = kwargs.get('choices', None)

evalscope/benchmarks/data_collection/data_collection_adapter.py

@@ -14,6 +14,7 @@ logger = get_logger()
 @Benchmark.register(
     name='data_collection',
     dataset_id='',  # dataset_id need to be set
+    description='Data collection',
     subset_list=['default'],
     metric_list=['AverageAccuracy'],
     few_shot_num=0,

evalscope/benchmarks/docmath/docmath_adapter.py

@@ -16,6 +16,7 @@ Format your response as follows: "Therefore, the answer is (insert answer here)"
 @Benchmark.register(
     name='docmath',
     pretty_name='DocMath',
+    tags=['Reasoning', 'Mathematics', 'Long Context'],
     description=
     'DocMath-Eval is a comprehensive benchmark focused on numerical reasoning within specialized domains. It requires the model to comprehend long and specialized documents and perform numerical reasoning to answer the given question.',  # noqa: E501
     dataset_id='yale-nlp/DocMath-Eval',

evalscope/benchmarks/drop/drop_adapter.py

@@ -31,6 +31,9 @@ Answer:  43
 @Benchmark.register(
     name='drop',
     pretty_name='DROP',
+    tags=['Reasoning'],
+    description=
+    'The DROP (Discrete Reasoning Over Paragraphs) benchmark is designed to evaluate the reading comprehension and reasoning capabilities of AI models. It includes a variety of tasks that require models to read passages and answer questions based on the content.',  # noqa: E501
     dataset_id='AI-ModelScope/DROP',
     metric_list=['AverageAccuracy'],
     few_shot_num=0,

evalscope/benchmarks/frames/frames_adapter.py

@@ -16,6 +16,7 @@ Format your response as follows: "Therefore, the answer is (insert answer here)"
 @Benchmark.register(
     name='frames',
     pretty_name='FRAMES',
+    tags=['Reasoning', 'Long Context'],
     description=
     'FRAMES is a comprehensive evaluation dataset designed to test the capabilities of Retrieval-Augmented Generation (RAG) systems across factuality, retrieval accuracy, and reasoning.',  # noqa: E501
     dataset_id='iic/frames',