evalscope 0.16.0__py3-none-any.whl → 0.16.2__py3-none-any.whl

evalscope/benchmarks/docmath/docmath_adapter.py

@@ -0,0 +1,85 @@
+from evalscope.benchmarks import Benchmark, DataAdapter
+from evalscope.constants import EvalType
+from evalscope.metrics import LLMJudge
+
+TEMPLATE_0SHOT = """Please read the following text and answer the question below.
+
+<text>
+{context}
+</text>
+
+{question}
+
+Format your response as follows: "Therefore, the answer is (insert answer here)"."""
+
+
+@Benchmark.register(
+    name='docmath',
+    pretty_name='DocMath',
+    tags=['Reasoning', 'Mathematics', 'Long Context'],
+    description=
+    'DocMath-Eval is a comprehensive benchmark focused on numerical reasoning within specialized domains. It requires the model to comprehend long and specialized documents and perform numerical reasoning to answer the given question.',  # noqa: E501
+    dataset_id='yale-nlp/DocMath-Eval',
+    metric_list=['AverageAccuracy'],
+    subset_list=['complong_testmini', 'compshort_testmini', 'simplong_testmini', 'simpshort_testmini'],
+    few_shot_num=0,
+    train_split=None,
+    eval_split='test',
+    prompt_template=TEMPLATE_0SHOT,
+)
+class DocMathAdapter(DataAdapter):
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    def load(self, **kwargs):
+        # default load mini test
+        kwargs['split_as_subset'] = True
+        data_dict = super().load(**kwargs)
+        return data_dict
+
+    def gen_prompt(self, input_d: dict, subset_name: str, few_shot_list: list, **kwargs) -> dict:
+        """
+        Generate model prompt from input data.
+        """
+        context = context = '\n'.join(input_d['paragraphs'])
+        question = input_d['question']
+        prompt = self.prompt_template.format(context=context, question=question)
+        return self.gen_prompt_data(prompt)
+
+    def get_gold_answer(self, input_d: dict) -> str:
+        """
+        Parse the raw input labels (gold).
+        """
+        return input_d['ground_truth']
+
+    def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = EvalType.CHECKPOINT) -> str:
+        """
+        Parse the predicted result and extract proper answer.
+        """
+        from .utils import extract_answer
+
+        extracted_answer = extract_answer(result)
+        return extracted_answer
+
+    def match(self, gold: str, pred: str) -> float:
+        """
+        Match the gold answer and the predicted answer.
+        """
+        from .utils import get_acc
+
+        return get_acc(prediction=pred, gt=gold)
+
+    def llm_match(self, gold: str, pred: str, judge: LLMJudge, **kwargs) -> float:
+        from .utils import GENERAL_ORM_PROMPT, ORM_USER_TEMPLATE
+
+        raw_input = kwargs.get('raw_input', None)
+        question = raw_input['question']
+        # get grading response
+        prompt = ORM_USER_TEMPLATE.format(problem=question, answer_1=gold, answer_2=pred)
+        orm_response = judge(prompt=prompt, system_prompt=GENERAL_ORM_PROMPT)
+        # parse grading response
+        if 'YES' in orm_response:
+            return 1.0
+        else:
+            return 0.0

evalscope/benchmarks/docmath/utils.py

@@ -0,0 +1,220 @@
+import math
+import numpy as np
+import re
+from sympy import Rational
+
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+GENERAL_ORM_PROMPT = """You are an expert in verifying if two answers are the same.
+Your input is a problem and two answers, Answer 1 and Answer 2. You need to check if they are equivalent.
+Your task is to determine if two answers are equivalent, without attempting to solve the original problem.
+Compare the answers to verify they represent identical values or meaning, even when written in different forms or notations.
+
+Your output must follow the following format:
+1) Provide an explanation for why the answers are equivalent or not.
+2) Then provide your final answer in the form of: [[YES]] or [[NO]]
+"""  # noqa: E501
+
+ORM_USER_TEMPLATE = """
+Problem: {problem}
+Answer 1: {answer_1}
+Answer 2: {answer_2}
+"""
+
+
+def round_up_to_decimal(number, decimals):
+    factor = 10**decimals
+    return math.ceil(number * factor) / factor
+
+
+def is_number(string):
+    pattern = r'^[-+]?(\d{1,3}(,\d{3})*|(\d+))(\.\d+)?$'
+    match = re.match(pattern, string)
+    return bool(match)
+
+
+def is_scientific_number(string):
+    pattern = r'^[-+]?\d+(\.\d+)?e[-]?\d+$'
+    match = re.match(pattern, string)
+    return bool(match)
+
+
+def normalize(prediction: str):
+    # Preprocessing the string [Stage 1]
+    prediction = prediction.strip()
+    prediction = prediction.rstrip('.')
+    if not isinstance(prediction, str):
+        prediction = str(prediction) if prediction is not None else '0'
+
+    for money in ['£', '€', '¥', 'million', 'billion', 'thousand', 'US', 'USD', 'RMB']:
+        prediction = prediction.replace(money, '')
+
+    # Replace special tokens
+    if '=' in prediction:
+        prediction = prediction.split('=')[-1].strip()
+    if '≈' in prediction:
+        prediction = prediction.split('≈')[-1].strip()
+    if '`' in prediction:
+        prediction = prediction.replace('`', '')
+    if '%' in prediction:
+        prediction = prediction.replace('%', '')
+    if '$' in prediction:
+        prediction = prediction.replace('$', '')
+    if '°' in prediction:
+        prediction = prediction.replace('°', '')
+
+    # Detect the boolean keyword in the generation
+    if prediction in ['true', 'yes', 'false', 'no']:
+        if prediction == 'true' or prediction == 'yes':
+            prediction = 'True'
+        else:
+            prediction = 'False'
+    if 'True' in prediction or 'False' in prediction:
+        prediction = 'True' if 'True' in prediction else 'False'
+
+    # Detect the approximation keyword
+    if 'approximately' in prediction:
+        prediction = prediction.replace('approximately', '').strip()
+    if ' or ' in prediction:
+        prediction = prediction.split(' or ')[0]
+
+    # Drop the units before and after the number
+    if re.match(r'[-+]?(?:[\d,]*\.*\d+) [^0-9 ]+$', prediction):
+        prediction = re.search(r'([-+]?(?:[\d,]*\.*\d+)) [^0-9 ]+$', prediction).group(1)
+    if re.match(r'[^0-9 ]+ [-+]?(?:[\d,]*\.*\d+)$', prediction):
+        prediction = re.search(r'[^0-9 ]+ ([-+]?(?:[\d,]*\.*\d+))$', prediction).group(1)
+    if re.match(r'[-+]?(?:[\d,]*\.*\d+)[^\d]{1,2}$', prediction):
+        prediction = re.search(r'([-+]?(?:[\d,]*\.*\d+))[^\d]{1,2}$', prediction).group(1)
+    if re.match(r'[^-+\d]{1,2}(?:[\d,]*\.*\d+)$', prediction):
+        prediction = re.search(r'[^-+\d]{1,2}((?:[\d,]*\.*\d+))$', prediction).group(1)
+
+    # Preprocessing the number [Stage 1]
+    if '10^' in prediction:
+        prediction = re.sub(r'10\^(-?\d+)', r'math.pow(10, \1)', prediction)
+    if ' x ' in prediction:
+        prediction = prediction.replace(' x ', '*')
+    if ' × ' in prediction:
+        prediction = prediction.replace(' × ', '*')
+    if is_number(prediction):
+        prediction = prediction.replace(',', '')
+
+    # Preprocessing the option [Stage 3]
+    if '(a)' in prediction or '(b)' in prediction or '(c)' in prediction or '(d)' in prediction:
+        prediction = '"' + re.search(r'\([a-d]\)', prediction).group(0) + '"'
+
+    # If the prediction is empty, use dummy '0'
+    if not prediction:
+        prediction = '0'
+
+    # Converting the string answer to a number/list/bool/option
+    try:
+        prediction = eval(prediction)
+    except Exception:
+        # TO CHECK
+        prediction = 0
+
+    # Performing common type conversion
+    if isinstance(prediction, (set, tuple)):
+        prediction = list(prediction)
+        if isinstance(prediction[0], complex):
+            prediction = [tmp.real for tmp in prediction]
+        elif isinstance(prediction[0], Rational):
+            prediction = [float(tmp) for tmp in prediction]
+    elif isinstance(prediction, np.ndarray):
+        prediction = prediction.tolist()
+    else:
+        if isinstance(prediction, complex):
+            prediction = prediction.real
+        elif isinstance(prediction, Rational):
+            prediction = float(prediction)
+
+    return prediction
+
+
+def extract_answer(response: str):
+    """Parses the final answer from the model's response text.
+
+    Args:
+        response: Text extracted from the model's response
+
+    Returns:
+        The final answer as a numeric value (string), or None if not found
+    """
+    # Remove any asterisks or other unwanted characters
+    response = response.replace('*', '')
+    response = response.replace('(', '')
+    response = response.replace(')', '')
+
+    # Search for the pattern 'the answer is {final answer}.'
+    match = re.search(r'the answer is (\=?\≈?\`?\%?\$?\°?\£?\€?\¥?-?[0-9\.,]+)', response, re.IGNORECASE)
+
+    if match:
+        # Remove commas from the matched number (if any)
+        res = match.group(1).replace(',', '').rstrip('.')
+        return res
+    else:
+        return response
+
+
+def within_eps(pred: float, gt: float):
+    eps = abs(gt) * 0.0015
+    if pred >= gt - eps and pred <= gt + eps:
+        return True
+    else:
+        return False
+
+
+def compare_two_numbers(p, gt):
+    if isinstance(p, int) or isinstance(p, float):
+        pass
+    elif isinstance(p, list) or isinstance(p, bool) or isinstance(p, str):
+        return False
+    elif isinstance(p, tuple) or isinstance(p, complex) or isinstance(p, dict):
+        return False
+    else:
+        raise ValueError(p)
+
+    v1, v2 = max(abs(gt), abs(p)), min(abs(gt), abs(p))
+    if (v1 != 0 and v2 != 0) and int(math.log10(v1 / v2)) == math.log10(v1 / v2):
+        return True
+
+    if v2 <= v1 / 50 and within_eps(pred=v2 * 100, gt=v1):
+        return True
+    elif v2 <= v1 / 500 and within_eps(pred=v2 * 1000, gt=v1):
+        return True
+    elif v2 <= v1 / 50000 and within_eps(pred=v2 * 100000, gt=v1):
+        return True
+
+    if round_up_to_decimal(v1, 2) == round_up_to_decimal(v2, 2):
+        return True
+
+    return within_eps(pred=p, gt=gt)
+
+
+def get_acc(prediction, gt, cot=True):
+    try:
+        if cot:
+            prediction = normalize(prediction)
+        else:
+            prediction = float(prediction)
+
+        answer_type = type(gt).__name__
+        assert answer_type in ['int', 'float', 'float64', 'bool'], answer_type
+        if isinstance(prediction, (str, int, float, bool)) or isinstance(prediction, list):
+            # Comparing prediction against the reference
+            if answer_type in ['bool']:
+                acc = int(prediction == gt)
+            elif answer_type == 'int':
+                acc = int(compare_two_numbers(prediction, gt))
+            elif answer_type == 'float' or answer_type == 'float64':
+                acc = int(compare_two_numbers(prediction, gt))
+            else:
+                acc = 0
+        else:
+            acc = 0
+            logger.error('Error: ', prediction, type(prediction))
+        return acc
+    except Exception:
+        return 0

evalscope/benchmarks/drop/drop_adapter.py

@@ -31,6 +31,9 @@ Answer:  43
 @Benchmark.register(
     name='drop',
     pretty_name='DROP',
+    tags=['Reasoning'],
+    description=
+    'The DROP (Discrete Reasoning Over Paragraphs) benchmark is designed to evaluate the reading comprehension and reasoning capabilities of AI models. It includes a variety of tasks that require models to read passages and answer questions based on the content.',  # noqa: E501
     dataset_id='AI-ModelScope/DROP',
     metric_list=['AverageAccuracy'],
     few_shot_num=0,

evalscope/benchmarks/frames/frames_adapter.py

@@ -0,0 +1,91 @@
+from evalscope.benchmarks import Benchmark, DataAdapter
+from evalscope.constants import EvalType, OutputType
+from evalscope.metrics import LLMJudge, exact_match
+
+TEMPLATE_0SHOT = """Please read the following text and answer the question below.
+
+<text>
+{context}
+</text>
+
+{question}
+
+Format your response as follows: "Therefore, the answer is (insert answer here)"."""
+
+
+@Benchmark.register(
+    name='frames',
+    pretty_name='FRAMES',
+    tags=['Reasoning', 'Long Context'],
+    description=
+    'FRAMES is a comprehensive evaluation dataset designed to test the capabilities of Retrieval-Augmented Generation (RAG) systems across factuality, retrieval accuracy, and reasoning.',  # noqa: E501
+    dataset_id='iic/frames',
+    model_adapter=OutputType.GENERATION,
+    output_types=[OutputType.GENERATION],
+    metric_list=['AverageAccuracy'],
+    few_shot_num=0,
+    train_split=None,
+    eval_split='test',
+    prompt_template=TEMPLATE_0SHOT,
+)
+class FramesAdapter(DataAdapter):
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    def load(self, **kwargs):
+        # default load with snapshot
+        kwargs['file_structure'] = {'default': ['test.jsonl']}
+        data_dict = super().load_with_snapshot(**kwargs)
+        return data_dict
+
+    def gen_prompt(self, input_d: dict, subset_name: str, few_shot_list: list, **kwargs) -> dict:
+        """
+        Generate model prompt from input data.
+        """
+        context = '\n'.join([f"{i['title']}\n{i['text']}" for i in input_d['wiki_items']])
+        question = input_d['Prompt']
+        prompt = self.prompt_template.format(context=context, question=question)
+        return self.gen_prompt_data(prompt)
+
+    def get_gold_answer(self, input_d: dict) -> str:
+        """
+        Parse the raw input labels (gold).
+        """
+        return input_d['Answer']
+
+    def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = EvalType.CHECKPOINT) -> str:
+        """
+        Parse the predicted result and extract proper answer.
+        """
+        response = result.replace('*', '')
+
+        if 'the answer is' in response:
+            ans = response.rsplit('the answer is', 1)[-1].strip().strip('.').strip()
+        else:
+            ans = ''
+
+        return ans
+
+    def match(self, gold: str, pred: str) -> float:
+        """
+        Match the gold answer and the predicted answer.
+        """
+        from .utils import normalize_answer
+        gold = normalize_answer(gold)
+        pred = normalize_answer(pred)
+        return exact_match(gold=gold, pred=pred)
+
+    def llm_match(self, gold: str, pred: str, judge: LLMJudge, **kwargs) -> float:
+        from .utils import GENERAL_ORM_PROMPT, ORM_USER_TEMPLATE
+
+        raw_input = kwargs.get('raw_input', None)
+        question = raw_input['Prompt']
+        # get grading response
+        prompt = ORM_USER_TEMPLATE.format(problem=question, answer_1=gold, answer_2=pred)
+        orm_response = judge(prompt=prompt, system_prompt=GENERAL_ORM_PROMPT)
+        # parse grading response
+        if 'YES' in orm_response:
+            return 1.0
+        else:
+            return 0.0

evalscope/benchmarks/frames/utils.py

@@ -0,0 +1,37 @@
+import re
+import string
+
+
+def normalize_answer(s):
+
+    def remove_articles(text):
+        return re.sub(r'\b(a|an|the)\b', ' ', text)
+
+    def white_space_fix(text):
+        return ' '.join(text.split())
+
+    def remove_punc(text):
+        exclude = set(string.punctuation)
+        return ''.join(ch for ch in text if ch not in exclude)
+
+    def lower(text):
+        return text.lower()
+
+    return white_space_fix(remove_articles(remove_punc(lower(s))))
+
+
+GENERAL_ORM_PROMPT = """You are an expert in verifying if two answers are the same.
+Your input is a problem and two answers, Answer 1 and Answer 2. You need to check if they are equivalent.
+Your task is to determine if two answers are equivalent, without attempting to solve the original problem.
+Compare the answers to verify they represent identical values or meaning, even when written in different forms or notations.
+
+Your output must follow the following format:
+1) Provide an explanation for why the answers are equivalent or not.
+2) Then provide your final answer in the form of: [[YES]] or [[NO]]
+"""  # noqa: E501
+
+ORM_USER_TEMPLATE = """
+Problem: {problem}
+Answer 1: {answer_1}
+Answer 2: {answer_2}
+"""

evalscope/benchmarks/general_mcq/general_mcq_adapter.py

@@ -1,11 +1,12 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
-import csv
 import os
+from collections import defaultdict
 
 from evalscope.benchmarks import Benchmark, DataAdapter
 from evalscope.constants import EvalType, OutputType
 from evalscope.metrics import exact_match
 from evalscope.utils import ResponseParser
+from evalscope.utils.io_utils import csv_to_list, jsonl_to_list
 from evalscope.utils.logger import get_logger
 
 # flake8: noqa
@@ -15,7 +16,9 @@ logger = get_logger()
 
 @Benchmark.register(
     name='general_mcq',
-    pretty_name='General MCQ',
+    pretty_name='General-MCQ',
+    description='A general multiple-choice question answering dataset.',
+    tags=['MCQ', 'Custom'],
     dataset_id='general_mcq',
     model_adapter=OutputType.GENERATION,
     output_types=[OutputType.MULTIPLE_CHOICE, OutputType.GENERATION],
@@ -24,7 +27,7 @@ logger = get_logger()
     few_shot_num=0,
     train_split='dev',
     eval_split='val',
-    prompt_template='请回答问题，并选出其中的正确答案\n{query}',
+    prompt_template='请回答问题，并选出其中的正确答案。你的回答的最后一行应该是这样的格式：“答案是：LETTER”（不带引号），其中 LETTER 是 A、B、C、D 中的一个。\n{query}',
     query_template='问题：{question}\n{choices}\n答案: {answer}\n\n')
 class GeneralMCQAdapter(DataAdapter):
 
@@ -34,28 +37,21 @@ class GeneralMCQAdapter(DataAdapter):
         self.choices = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J']
 
     def load_from_disk(self, dataset_name_or_path, subset_list, work_dir, **kwargs) -> dict:
-        data_dict = {}
+        data_dict = defaultdict(dict)
         for subset_name in subset_list:
             for split_name in [self.train_split, self.eval_split]:
-                if os.path.exists(dataset_name_or_path):
-                    file_path = os.path.join(dataset_name_or_path, f'{subset_name}_{split_name}.csv')
-                else:
-                    file_path = os.path.join(work_dir, dataset_name_or_path, f'{subset_name}_{split_name}.csv')
-                if os.path.exists(file_path):
-                    with open(file_path, encoding='utf-8') as f:
-                        rows = []
-                        reader = csv.reader(f)
-                        header = next(reader)
-                        for row in reader:
-                            item = dict(zip(header, row))
-                            rows.append(item)
-
-                        if subset_name in data_dict:
-                            data_dict[subset_name].update({split_name: rows})
-                        else:
-                            data_dict[subset_name] = {split_name: rows}
-
-        return data_dict
+                # Check for files with different extensions
+                for ext, loader in [('.jsonl', jsonl_to_list), ('.csv', csv_to_list)]:
+                    if os.path.exists(dataset_name_or_path):
+                        file_path = os.path.join(dataset_name_or_path, f'{subset_name}_{split_name}{ext}')
+                    else:
+                        file_path = os.path.join(work_dir, dataset_name_or_path, f'{subset_name}_{split_name}{ext}')
+
+                    if os.path.exists(file_path):
+                        data_dict[subset_name][split_name] = loader(file_path)
+                        break  # Stop checking other extensions once a file is found
+
+        return dict(data_dict)
 
     def gen_prompt(self, input_d: dict, subset_name: str, few_shot_list: list, **kwargs) -> dict:
         """

evalscope/benchmarks/general_qa/general_qa_adapter.py

@@ -13,6 +13,9 @@ logger = get_logger()
 
 @Benchmark.register(
     name='general_qa',
+    pretty_name='General-QA',
+    description='General Question Answering dataset',
+    tags=['QA', 'Custom'],
     dataset_id='general_qa',
     subset_list=['default'],
     metric_list=['AverageBLEU', 'AverageRouge'],

evalscope/benchmarks/gpqa/gpqa_adapter.py

@@ -10,6 +10,9 @@ from evalscope.metrics import exact_match
 @Benchmark.register(
     name='gpqa',
     pretty_name='GPQA',
+    tags=['MCQ', 'Knowledge'],
+    description=
+    'GPQA is a dataset for evaluating the reasoning ability of large language models (LLMs) on complex mathematical problems. It contains questions that require step-by-step reasoning to arrive at the correct answer.',  # noqa: E501
     dataset_id='modelscope/gpqa',
     model_adapter=OutputType.GENERATION,
     output_types=[OutputType.MULTIPLE_CHOICE, OutputType.GENERATION],

evalscope/benchmarks/gsm8k/gsm8k_adapter.py

@@ -15,6 +15,9 @@ logger = get_logger()
 @Benchmark.register(
     name='gsm8k',
     pretty_name='GSM8K',
+    tags=['Mathematics'],
+    description=
+    'GSM8K (Grade School Math 8K) is a dataset of grade school math problems, designed to evaluate the mathematical reasoning abilities of AI models.',
     dataset_id='modelscope/gsm8k',
     subset_list=['main'],
     metric_list=['AverageAccuracy'],

evalscope/benchmarks/hellaswag/hellaswag_adapter.py

@@ -18,6 +18,9 @@ logger = get_logger()
 @Benchmark.register(
     name='hellaswag',
     pretty_name='HellaSwag',
+    tags=['Commonsense', 'MCQ', 'Knowledge'],
+    description=
+    'HellaSwag is a benchmark for commonsense reasoning in natural language understanding tasks. It consists of multiple-choice questions where the model must select the most plausible continuation of a given context.',
     dataset_id='modelscope/hellaswag',
     model_adapter=OutputType.MULTIPLE_CHOICE,
     output_types=[OutputType.MULTIPLE_CHOICE, OutputType.GENERATION],

evalscope/benchmarks/humaneval/humaneval_adapter.py

@@ -13,6 +13,9 @@ logger = get_logger()
 @Benchmark.register(
     name='humaneval',
     pretty_name='HumanEval',
+    tags=['Coding'],
+    description=
+    'HumanEval is a benchmark for evaluating the ability of code generation models to write Python functions based on given specifications. It consists of programming tasks with a defined input-output behavior.',  # noqa: E501
     dataset_id='modelscope/humaneval',
     subset_list=['openai_humaneval'],
     metric_list=['Pass@1'],

evalscope/benchmarks/ifeval/ifeval_adapter.py

@@ -10,6 +10,9 @@ from evalscope.metrics import Metric, mean, metric_registry
 @Benchmark.register(
     name='ifeval',
     pretty_name='IFEval',
+    tags=['Instruction-Following'],
+    description=
+    'IFEval is a benchmark for evaluating instruction-following language models, focusing on their ability to understand and respond to various prompts. It includes a diverse set of tasks and metrics to assess model performance comprehensively.',  # noqa: E501
     dataset_id='opencompass/ifeval',
     subset_list=['default'],
     metric_list=[

evalscope/benchmarks/iquiz/iquiz_adapter.py

@@ -7,6 +7,9 @@ from evalscope.utils.utils import ResponseParser
 @Benchmark.register(
     name='iquiz',
     pretty_name='IQuiz',
+    tags=['Knowledge', 'MCQ', 'Chinese'],
+    description=
+    'IQuiz is a benchmark for evaluating AI models on IQ and EQ questions. It consists of multiple-choice questions where the model must select the correct answer and provide an explanation.',  # noqa: E501
     dataset_id='AI-ModelScope/IQuiz',
     model_adapter=OutputType.GENERATION,
     output_types=[OutputType.MULTIPLE_CHOICE, OutputType.GENERATION],

evalscope/benchmarks/live_code_bench/live_code_bench_adapter.py

@@ -8,7 +8,10 @@ logger = get_logger()
 
 @Benchmark.register(
     name='live_code_bench',
-    pretty_name='Live Code Bench',
+    pretty_name='Live-Code-Bench',
+    tags=['Coding'],
+    description=
+    'Live Code Bench is a benchmark for evaluating code generation models on real-world coding tasks. It includes a variety of programming problems with test cases to assess the model\'s ability to generate correct and efficient code solutions.',  # noqa: E501
     dataset_id='AI-ModelScope/code_generation_lite',
     subset_list=['release_latest'],
     metric_list=['Pass@1'],

evalscope/benchmarks/maritime_bench/maritime_bench_adapter.py

@@ -11,6 +11,9 @@ SUBSET_LIST = ['default']
 @Benchmark.register(
     name='maritime_bench',
     pretty_name='MaritimeBench',
+    tags=['Maritime', 'MCQ', 'Knowledge'],
+    description=
+    'MaritimeBench is a benchmark for evaluating AI models on maritime-related multiple-choice questions. It consists of questions related to maritime knowledge, where the model must select the correct answer from given options.',  # noqa: E501
     dataset_id='HiDolphin/MaritimeBench',
     model_adapter=OutputType.GENERATION,
     output_types=[OutputType.MULTIPLE_CHOICE, OutputType.GENERATION],

evalscope/benchmarks/math_500/math_500_adapter.py

@@ -10,6 +10,9 @@ logger = get_logger()
 @Benchmark.register(
     name='math_500',
     pretty_name='MATH-500',
+    tags=['Mathematics'],
+    description=
+    "MATH-500 is a benchmark for evaluating mathematical reasoning capabilities of AI models. It consists of 500 diverse math problems across five levels of difficulty, designed to test a model's ability to solve complex mathematical problems by generating step-by-step solutions and providing the correct final answer.",  # noqa: E501
     dataset_id='AI-ModelScope/MATH-500',
     subset_list=['Level 1', 'Level 2', 'Level 3', 'Level 4', 'Level 5'],
     metric_list=['AveragePass@1'],

evalscope/benchmarks/mmlu/mmlu_adapter.py

@@ -136,6 +136,9 @@ SUBJECT_MAPPING = {
 @Benchmark.register(
     name='mmlu',
     pretty_name='MMLU',
+    tags=['Knowledge', 'MCQ'],
+    description=
+    "The MMLU (Massive Multitask Language Understanding) benchmark is a comprehensive evaluation suite designed to assess the performance of language models across a wide range of subjects and tasks. It includes multiple-choice questions from various domains, such as history, science, mathematics, and more, providing a robust measure of a model's understanding and reasoning capabilities.",  # noqa: E501
     dataset_id='modelscope/mmlu',
     model_adapter=OutputType.GENERATION,
     output_types=[OutputType.MULTIPLE_CHOICE, OutputType.GENERATION],

evalscope/benchmarks/mmlu_pro/mmlu_pro_adapter.py

@@ -15,6 +15,9 @@ SUBSET_LIST = [
 @Benchmark.register(
     name='mmlu_pro',
     pretty_name='MMLU-Pro',
+    tags=['MCQ', 'Knowledge'],
+    description=
+    'MMLU-Pro is a benchmark for evaluating language models on multiple-choice questions across various subjects. It includes questions from different domains, where the model must select the correct answer from given options.',  # noqa: E501
     dataset_id='modelscope/MMLU-Pro',
     model_adapter=OutputType.GENERATION,
     output_types=[OutputType.MULTIPLE_CHOICE, OutputType.GENERATION],