evalscope 1.0.0__py3-none-any.whl → 1.0.1__py3-none-any.whl

evalscope/report/generator.py

@@ -8,105 +8,26 @@ from evalscope.report.report import *
 if TYPE_CHECKING:
     from evalscope.api.benchmark import DataAdapter
     from evalscope.api.metric import AggScore
-    from evalscope.benchmarks import DataAdapter as OldDataAdapter
 
 
 class ReportGenerator:
 
     @staticmethod
-    def gen_report(subset_score_map: dict, model_name: str, data_adapter: 'OldDataAdapter', **kwargs) -> Report:
-        """
-        Generate a report for a specific dataset based on provided subset scores.
-
-        Args:
-            subset_score_map (dict): A mapping from subset names to a list of score dictionaries.
-                    {
-                        'subset_name': [
-                            {'metric_name': 'AverageAccuracy', 'score': 0.3389, 'num': 100},
-                            {'metric_name': 'WeightedAverageAccuracy', 'score': 0.3389, 'num': 100}
-                        ],
-                        ...
-                    }
-            report_name (str): The name of the report to generate.
-            data_adapter (DataAdapter): An adapter object for data handling.
-
-        Returns:
-            Report: A structured report object containing metrics, categories, and subsets.
-
-            >>> report = gen_report(subset_score_map, "My Report", data_adapter, dataset_name="Dataset", model_name="Model")
-        """  # noqa: E501
-
-        dataset_name = data_adapter.name
-        category_map = data_adapter.category_map
-        report_name = f'{model_name}@{dataset_name}'
-
-        def flatten_subset() -> DataFrame:
-            """
-            Flatten subset score map to a DataFrame.
-
-            Example:
-                        name  score  num   categories      metric_name
-            0       ARC-Easy    0.5    2    [default]  AverageAccuracy
-            1  ARC-Challenge    0.5    2    [default]  AverageAccuracy
-            """
-            subsets = []
-            for subset_name, scores in subset_score_map.items():
-                for score_item in scores:
-                    categories = category_map.get(subset_name, ['default'])
-                    if isinstance(categories, str):
-                        categories = [categories]
-                    subsets.append(
-                        dict(
-                            name=subset_name,
-                            score=score_item['score'],
-                            num=score_item['num'],
-                            metric_name=score_item['metric_name'],
-                            categories=tuple(categories)
-                        )
-                    )
-            df = pd.DataFrame(subsets)
-            return df
-
-        df = flatten_subset()
-
+    def gen_collection_report(df: DataFrame, all_dataset_name: str, model_name: str) -> Report:
         metrics_list = []
-        for metric_name, group_metric in df.groupby('metric_name', sort=False):
+        for metric_name, group_metric in df.groupby('metric', sort=False):
             categories = []
             for category_name, group_category in group_metric.groupby('categories'):
                 subsets = []
-                for _, row in group_category.iterrows():
-                    subsets.append(Subset(name=row['name'], score=row['score'], num=row['num']))
-
+                for (dataset_name, subset_name), group_subset in group_category.groupby(['dataset_name',
+                                                                                         'subset_name']):
+                    avg_score = group_subset['score'].mean()
+                    num = group_subset['score'].count()
+                    subsets.append(Subset(name=f'{dataset_name}/{subset_name}', score=float(avg_score), num=int(num)))
                 categories.append(Category(name=category_name, subsets=subsets))
-
             metrics_list.append(Metric(name=metric_name, categories=categories))
-
-        report = Report(
-            name=report_name,
-            metrics=metrics_list,
-            dataset_name=dataset_name,
-            model_name=model_name,
-            dataset_description=data_adapter.description,
-            dataset_pretty_name=data_adapter.pretty_name
-        )
-        return report
-
-    @staticmethod
-    def gen_collection_report(df: DataFrame, all_dataset_name: str, model_name: str) -> Report:
-        categories = []
-        for category_name, group_category in df.groupby('categories'):
-            subsets = []
-            for (dataset_name, subset_name), group_subset in group_category.groupby(['dataset_name', 'subset_name']):
-                avg_score = group_subset['score'].mean()
-                num = group_subset['score'].count()
-                subsets.append(Subset(name=f'{dataset_name}/{subset_name}', score=float(avg_score), num=int(num)))
-
-            categories.append(Category(name=category_name, subsets=subsets))
         return Report(
-            name=DataCollection.NAME,
-            metrics=[Metric(name='Average', categories=categories)],
-            dataset_name=all_dataset_name,
-            model_name=model_name
+            name=DataCollection.NAME, metrics=metrics_list, dataset_name=all_dataset_name, model_name=model_name
         )
 
     @staticmethod

evalscope/run.py

@@ -131,8 +131,9 @@ def evaluate_model(task_config: TaskConfig, outputs: OutputsStructure) -> dict:
         )
         evaluators.append(evaluator)
 
-        # Update task_config.dataset_args with benchmark metadata
-        task_config.dataset_args[dataset_name] = benchmark.to_dict()
+        # Update task_config.dataset_args with benchmark metadata, except for DataCollection
+        if dataset_name != DataCollection.NAME:
+            task_config.dataset_args[dataset_name] = benchmark.to_dict()
 
     # dump task_cfg to outputs.configs_dir after creating evaluators
     task_config.dump_yaml(outputs.configs_dir)
@@ -149,17 +150,20 @@ def evaluate_model(task_config: TaskConfig, outputs: OutputsStructure) -> dict:
         logger.info(f'Overall report table: \n{report_table} \n')
     except Exception:
         logger.error('Failed to generate report table.')
-
     # Clean up
     if model is not None:
         import gc
-        import torch
 
         del model
         del evaluators
-        torch.cuda.empty_cache()
         gc.collect()
 
+        from evalscope.utils.import_utils import check_import
+        if check_import('torch'):
+            import torch
+            if torch.cuda.is_available():
+                torch.cuda.empty_cache()
+
     return eval_results
 
 

evalscope/third_party/toolbench_static/llm/swift_infer.py

@@ -1,9 +1,5 @@
-import os
 from dataclasses import dataclass
-from swift.llm import InferEngine, InferRequest, PtEngine, RequestConfig, get_template
 
-# 设置GPU环境变量
-os.environ['CUDA_VISIBLE_DEVICES'] = '0'
 
 @dataclass
 class SwiftInferArgs:

evalscope/utils/chat_service.py

@@ -1,6 +1,5 @@
 import os
 import time
-import torch
 from contextlib import contextmanager
 from functools import partial
 from pydantic import BaseModel, Field
@@ -95,6 +94,7 @@ class TextCompletionResponse(BaseModel):
 class ChatService:
 
     def __init__(self, model_path, attn_implementation):
+        import torch
         from modelscope import AutoModelForCausalLM, AutoTokenizer
         from transformers import TextIteratorStreamer
 

evalscope/utils/import_utils.py

@@ -5,13 +5,35 @@ import importlib
 import os
 from itertools import chain
 from types import ModuleType
-from typing import Any
+from typing import Any, Optional, Union
 
 from .logger import get_logger
 
 logger = get_logger()  # pylint: disable=invalid-name
 
 
+def check_import(module_name: str, package: Optional[str] = None, raise_error: bool = False) -> bool:
+    """Check if a module can be imported.
+
+    Args:
+        module_name (str): The name of the module to check.
+        package (str, optional): The package to install if the module is not found. Defaults to None.
+        raise_error (bool, optional): Whether to raise an error if the module is not found. Defaults to False.
+    """
+    try:
+        importlib.import_module(module_name)
+        return True
+    except ImportError:
+        error_msg = f'`{module_name}` not found.'
+        if package:
+            error_msg += f' Please run `pip install {package}` to use this feature.'
+            logger.warning(error_msg)
+
+        if raise_error:
+            raise ImportError(error_msg)
+        return False
+
+
 class _LazyModule(ModuleType):
     """
     Module class that surfaces all objects but only performs associated imports when the objects are requested.

evalscope/utils/io_utils.py

@@ -1,6 +1,7 @@
 import base64
 import csv
 import hashlib
+import io
 import json
 import jsonlines as jsonl
 import os
@@ -283,22 +284,62 @@ def get_valid_list(input_list, candidate_list):
            [i for i in input_list if i not in candidate_list]
 
 
-def PIL_to_base64(image: Image.Image, format: str = 'JPEG') -> str:
+def PIL_to_base64(image: Image.Image, format: str = 'JPEG', add_header: bool = False) -> str:
     """
     Convert a PIL Image to a base64 encoded string.
 
     Args:
         image (Image.Image): The PIL Image to convert.
         format (str): The format to save the image in. Default is 'JPEG'.
+        add_header (bool): Whether to add the base64 header. Default is False.
+
     Returns:
         str: Base64 encoded string of the image.
     """
     buffered = BytesIO()
     image.save(buffered, format=format)
     img_str = base64.b64encode(buffered.getvalue()).decode('utf-8')
+    if add_header:
+        img_str = f'data:image/{format.lower()};base64,{img_str}'
+    return img_str
+
+
+def bytes_to_base64(bytes_data: bytes, format: str = 'png', add_header: bool = False) -> str:
+    """Convert image bytes to a base64 encoded string.
+
+    Args:
+        bytes_data (bytes): The bytes to convert.
+        add_header (bool): Whether to add the base64 header. Default is False.
+
+    Returns:
+        str: Base64 encoded string of the bytes.
+    """
+    img_str = base64.b64encode(bytes_data).decode('utf-8')
+    if add_header:
+        img_str = f'data:image/{format};base64,{img_str}'
     return img_str
 
 
+def base64_to_PIL(base64_str):
+    """Convert a base64 encoded string to a PIL Image.
+
+    Args:
+        base64_str (str): The base64 encoded string.
+
+    Returns:
+        Image.Image: The decoded PIL Image.
+    """
+    # remove header
+    if ',' in base64_str:
+        base64_str = base64_str.split(',', 1)[1]
+
+    # decode
+    img_data = base64.b64decode(base64_str)
+    img_file = io.BytesIO(img_data)
+    img = Image.open(img_file)
+    return img
+
+
 def safe_filename(s: str, max_length: int = 255) -> str:
     """
     Convert a string into a safe filename by removing or replacing unsafe characters.

evalscope/utils/model_utils.py

@@ -3,6 +3,8 @@ import random
 from enum import Enum
 from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, Union
 
+from evalscope.utils.import_utils import check_import
+
 if TYPE_CHECKING:
     from transformers import GenerationConfig
 
@@ -67,7 +69,8 @@ def seed_everything(seed: int):
     """
     random.seed(seed)
     np.random.seed(seed)
-    try:
+
+    if check_import('torch'):
         import torch
 
         torch.manual_seed(seed)
@@ -75,5 +78,3 @@ def seed_everything(seed: int):
             torch.cuda.manual_seed_all(seed)
             torch.backends.cudnn.deterministic = True
             torch.backends.cudnn.benchmark = False
-    except ImportError:
-        pass

evalscope/utils/multi_choices.py

@@ -1,11 +1,8 @@
 # flake8: noqa: E501
-from __future__ import annotations
-
 import re
-from typing import TYPE_CHECKING, List, Optional
+from typing import List, Optional, Union
 
-if TYPE_CHECKING:
-    from evalscope.api.evaluator import Choices, Target, TaskState
+from evalscope.api.evaluator import Choices, Target, TaskState
 
 FEW_SHOT_TEMPLATE = r"""Here are some examples of how to answer similar questions:
 
@@ -84,7 +81,9 @@ def answer_options(choices: Choices) -> str:
     return '\n'.join([f'{answer_character(i)}) {choices[j].value}' for i, j in enumerate(indexes)])
 
 
-def prompt(question: str, choices: Choices, template: str, fewshot: Optional[str] = None) -> str:
+def prompt(question: str, choices: Union[Choices, List[str]], template: str, fewshot: Optional[str] = None) -> str:
+    if isinstance(choices, list):
+        choices = Choices(choices)
 
     choices_text = answer_options(choices)
     letters = ','.join(answer_character(i) for i in range(len(choices)))
@@ -122,6 +121,14 @@ def format_example(
     return f'{question}\n{choices_text}\nANSWER: {answer.text}'
 
 
+def _fallback_parse_answer(completion: str) -> Optional[set[str]]:
+    # Fallback to find the last upper case letter
+    for letter in reversed(completion):
+        if letter.isupper():
+            return {letter}
+    return None
+
+
 def parse_answers(state: TaskState, multiple_correct: bool = False) -> set[str]:
     """
     Convenience function for extracting answers from the state output.
@@ -150,6 +157,11 @@ def parse_answers(state: TaskState, multiple_correct: bool = False) -> set[str]:
             state.output.completion,
         )
 
+    if match is None:
+        fallback_answer = _fallback_parse_answer(state.output.completion)
+        if fallback_answer:
+            return fallback_answer
+
     if match is None:
         return set()
 
@@ -200,6 +212,11 @@ def parse_answers_zh(state: TaskState, multiple_correct: bool = False) -> set[st
     pattern = r'答案\s*[:：]\s*([A-Za-z0-9,，]+)'
     match = re.search(pattern, state.output.completion, flags=re.MULTILINE)
 
+    if match is None:
+        fallback_answer = _fallback_parse_answer(state.output.completion)
+        if fallback_answer:
+            return fallback_answer
+
     if match is None:
         return set()
 

evalscope/version.py

@@ -1,4 +1,4 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 
-__version__ = '1.0.0'
-__release_datetime__ = '2025-08-25 12:00:00'
+__version__ = '1.0.1'
+__release_datetime__ = '2025-09-05 14:00:00'

{evalscope-1.0.0.dist-info → evalscope-1.0.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: evalscope
-Version: 1.0.0
+Version: 1.0.1
 Summary: EvalScope: Lightweight LLMs Evaluation Framework
 Home-page: https://github.com/modelscope/evalscope
 Author: ModelScope team
@@ -17,12 +17,10 @@ Classifier: Programming Language :: Python :: 3.12
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: accelerate
 Requires-Dist: colorlog
 Requires-Dist: datasets==3.6.0
 Requires-Dist: docstring-parser
 Requires-Dist: dotenv
-Requires-Dist: immutabledict
 Requires-Dist: jieba
 Requires-Dist: jsonlines
 Requires-Dist: langdetect
@@ -34,7 +32,6 @@ Requires-Dist: openai
 Requires-Dist: overrides
 Requires-Dist: pandas
 Requires-Dist: pillow
-Requires-Dist: pyarrow
 Requires-Dist: pydantic
 Requires-Dist: pyyaml>=5.1
 Requires-Dist: requests
@@ -46,7 +43,6 @@ Requires-Dist: scikit-learn
 Requires-Dist: seaborn
 Requires-Dist: sympy
 Requires-Dist: tabulate
-Requires-Dist: torch
 Requires-Dist: tqdm
 Requires-Dist: transformers>=4.33
 Requires-Dist: word2number
@@ -57,14 +53,13 @@ Requires-Dist: omegaconf; extra == "aigc"
 Requires-Dist: open-clip-torch; extra == "aigc"
 Requires-Dist: opencv-python; extra == "aigc"
 Requires-Dist: peft>=0.17; extra == "aigc"
+Requires-Dist: torch; extra == "aigc"
 Requires-Dist: torchvision; extra == "aigc"
 Provides-Extra: all
-Requires-Dist: accelerate; extra == "all"
 Requires-Dist: colorlog; extra == "all"
 Requires-Dist: datasets==3.6.0; extra == "all"
 Requires-Dist: docstring-parser; extra == "all"
 Requires-Dist: dotenv; extra == "all"
-Requires-Dist: immutabledict; extra == "all"
 Requires-Dist: jieba; extra == "all"
 Requires-Dist: jsonlines; extra == "all"
 Requires-Dist: langdetect; extra == "all"
@@ -76,7 +71,6 @@ Requires-Dist: openai; extra == "all"
 Requires-Dist: overrides; extra == "all"
 Requires-Dist: pandas; extra == "all"
 Requires-Dist: pillow; extra == "all"
-Requires-Dist: pyarrow; extra == "all"
 Requires-Dist: pydantic; extra == "all"
 Requires-Dist: pyyaml>=5.1; extra == "all"
 Requires-Dist: requests; extra == "all"
@@ -88,7 +82,6 @@ Requires-Dist: scikit-learn; extra == "all"
 Requires-Dist: seaborn; extra == "all"
 Requires-Dist: sympy; extra == "all"
 Requires-Dist: tabulate; extra == "all"
-Requires-Dist: torch; extra == "all"
 Requires-Dist: tqdm; extra == "all"
 Requires-Dist: transformers>=4.33; extra == "all"
 Requires-Dist: word2number; extra == "all"
@@ -100,6 +93,7 @@ Requires-Dist: langchain-core<0.4.0,>=0.3.0; extra == "all"
 Requires-Dist: langchain-openai<0.4.0,>=0.3.0; extra == "all"
 Requires-Dist: mteb==1.38.20; extra == "all"
 Requires-Dist: ragas==0.2.14; extra == "all"
+Requires-Dist: torch; extra == "all"
 Requires-Dist: webdataset>0.2.0; extra == "all"
 Requires-Dist: aiohttp; extra == "all"
 Requires-Dist: fastapi; extra == "all"
@@ -154,6 +148,7 @@ Requires-Dist: langchain-core<0.4.0,>=0.3.0; extra == "rag"
 Requires-Dist: langchain-openai<0.4.0,>=0.3.0; extra == "rag"
 Requires-Dist: mteb==1.38.20; extra == "rag"
 Requires-Dist: ragas==0.2.14; extra == "rag"
+Requires-Dist: torch; extra == "rag"
 Requires-Dist: webdataset>0.2.0; extra == "rag"
 Provides-Extra: vlmeval
 Requires-Dist: ms-vlmeval>=0.0.17; extra == "vlmeval"
@@ -277,7 +272,9 @@ Please scan the QR code below to join our community groups:
 >
 > Version 1.0 introduces a major overhaul of the evaluation framework, establishing a new, more modular and extensible API layer under `evalscope/api`. Key improvements include standardized data models for benchmarks, samples, and results; a registry-based design for components such as benchmarks and metrics; and a rewritten core evaluator that orchestrates the new architecture. Existing benchmark adapters have been migrated to this API, resulting in cleaner, more consistent, and easier-to-maintain implementations.
 
-- 🔥 **[2025.08.22]** Version 1.0 Refactoring.
+- 🔥 **[2025.09.05]** Added support for vision-language multimodal model evaluation tasks, such as MathVista and MMMU. For more supported datasets, please [refer to the documentation](https://evalscope.readthedocs.io/zh-cn/latest/get_started/supported_dataset/vlm.html).
+- 🔥 **[2025.09.04]** Added support for image editing task evaluation, including the [GEdit-Bench](https://modelscope.cn/datasets/stepfun-ai/GEdit-Bench) benchmark. For usage instructions, refer to the [documentation](https://evalscope.readthedocs.io/en/latest/user_guides/aigc/image_edit.html).
+- 🔥 **[2025.08.22]** Version 1.0 Refactoring. Break changes, please [refer to](https://evalscope.readthedocs.io/en/latest/get_started/basic_usage.html#switching-to-version-v1-0).
 - 🔥 **[2025.07.18]** The model stress testing now supports randomly generating image-text data for multimodal model evaluation. For usage instructions, refer to the [documentation](https://evalscope.readthedocs.io/en/latest/user_guides/stress_test/examples.html#id4).
 - 🔥 **[2025.07.16]** Support for [τ-bench](https://github.com/sierra-research/tau-bench) has been added, enabling the evaluation of AI Agent performance and reliability in real-world scenarios involving dynamic user and tool interactions. For usage instructions, please refer to the [documentation](https://evalscope.readthedocs.io/zh-cn/latest/get_started/supported_dataset/llm.html#bench).
 - 🔥 **[2025.07.14]** Support for "Humanity's Last Exam" ([Humanity's-Last-Exam](https://modelscope.cn/datasets/cais/hle)), a highly challenging evaluation benchmark. For usage instructions, refer to the [documentation](https://evalscope.readthedocs.io/en/latest/get_started/supported_dataset/llm.html#humanity-s-last-exam).
@@ -285,7 +282,7 @@ Please scan the QR code below to join our community groups:
 - 🔥 **[2025.06.28]** Optimized custom dataset evaluation: now supports evaluation without reference answers. Enhanced LLM judge usage, with built-in modes for "scoring directly without reference answers" and "checking answer consistency with reference answers". See [reference](https://evalscope.readthedocs.io/en/latest/advanced_guides/custom_dataset/llm.html#qa) for details.
 - 🔥 **[2025.06.19]** Added support for the [BFCL-v3](https://modelscope.cn/datasets/AI-ModelScope/bfcl_v3) benchmark, designed to evaluate model function-calling capabilities across various scenarios. For more information, refer to the [documentation](https://evalscope.readthedocs.io/zh-cn/latest/third_party/bfcl_v3.html).
 - 🔥 **[2025.06.02]** Added support for the Needle-in-a-Haystack test. Simply specify `needle_haystack` to conduct the test, and a corresponding heatmap will be generated in the `outputs/reports` folder, providing a visual representation of the model's performance. Refer to the [documentation](https://evalscope.readthedocs.io/en/latest/third_party/needle_haystack.html) for more details.
-- 🔥 **[2025.05.29]** Added support for two long document evaluation benchmarks: [DocMath](https://modelscope.cn/datasets/yale-nlp/DocMath-Eval/summary) and [FRAMES](https://modelscope.cn/datasets/iic/frames/summary). For usage guidelines, please refer to the [documentation](https://evalscope.readthedocs.io/en/latest/get_started/supported_dataset.html).
+- 🔥 **[2025.05.29]** Added support for two long document evaluation benchmarks: [DocMath](https://modelscope.cn/datasets/yale-nlp/DocMath-Eval/summary) and [FRAMES](https://modelscope.cn/datasets/iic/frames/summary). For usage guidelines, please refer to the [documentation](https://evalscope.readthedocs.io/en/latest/get_started/supported_dataset/index.html).
 - 🔥 **[2025.05.16]** Model service performance stress testing now supports setting various levels of concurrency and outputs a performance test report. [Reference example](https://evalscope.readthedocs.io/en/latest/user_guides/stress_test/quick_start.html#id3).
 - 🔥 **[2025.05.13]** Added support for the [ToolBench-Static](https://modelscope.cn/datasets/AI-ModelScope/ToolBench-Static) dataset to evaluate model's tool-calling capabilities. Refer to the [documentation](https://evalscope.readthedocs.io/en/latest/third_party/toolbench.html) for usage instructions. Also added support for the [DROP](https://modelscope.cn/datasets/AI-ModelScope/DROP/dataPeview) and [Winogrande](https://modelscope.cn/datasets/AI-ModelScope/winogrande_val) benchmarks to assess the reasoning capabilities of models.
 <details><summary>More</summary>
@@ -294,7 +291,7 @@ Please scan the QR code below to join our community groups:
 - 🔥 **[2025.04.27]** Support for text-to-image evaluation: Supports 8 metrics including MPS, HPSv2.1Score, etc., and evaluation benchmarks such as EvalMuse, GenAI-Bench. Refer to the [user documentation](https://evalscope.readthedocs.io/en/latest/user_guides/aigc/t2i.html) for more details.
 - 🔥 **[2025.04.10]** Model service stress testing tool now supports the `/v1/completions` endpoint (the default endpoint for vLLM benchmarking)
 - 🔥 **[2025.04.08]** Support for evaluating embedding model services compatible with the OpenAI API has been added. For more details, check the [user guide](https://evalscope.readthedocs.io/en/latest/user_guides/backend/rageval_backend/mteb.html#configure-evaluation-parameters).
-- 🔥 **[2025.03.27]** Added support for [AlpacaEval](https://www.modelscope.cn/datasets/AI-ModelScope/alpaca_eval/dataPeview) and [ArenaHard](https://modelscope.cn/datasets/AI-ModelScope/arena-hard-auto-v0.1/summary) evaluation benchmarks. For usage notes, please refer to the [documentation](https://evalscope.readthedocs.io/en/latest/get_started/supported_dataset.html)
+- 🔥 **[2025.03.27]** Added support for [AlpacaEval](https://www.modelscope.cn/datasets/AI-ModelScope/alpaca_eval/dataPeview) and [ArenaHard](https://modelscope.cn/datasets/AI-ModelScope/arena-hard-auto-v0.1/summary) evaluation benchmarks. For usage notes, please refer to the [documentation](https://evalscope.readthedocs.io/en/latest/get_started/supported_dataset/index.html)
 - 🔥 **[2025.03.20]** The model inference service stress testing now supports generating prompts of specified length using random values. Refer to the [user guide](https://evalscope.readthedocs.io/en/latest/user_guides/stress_test/examples.html#using-the-random-dataset) for more details.
 - 🔥 **[2025.03.13]** Added support for the [LiveCodeBench](https://www.modelscope.cn/datasets/AI-ModelScope/code_generation_lite/summary) code evaluation benchmark, which can be used by specifying `live_code_bench`. Supports evaluating QwQ-32B on LiveCodeBench, refer to the [best practices](https://evalscope.readthedocs.io/en/latest/best_practice/eval_qwq.html).
 - 🔥 **[2025.03.11]** Added support for the [SimpleQA](https://modelscope.cn/datasets/AI-ModelScope/SimpleQA/summary) and [Chinese SimpleQA](https://modelscope.cn/datasets/AI-ModelScope/Chinese-SimpleQA/summary) evaluation benchmarks. These are used to assess the factual accuracy of models, and you can specify `simple_qa` and `chinese_simpleqa` for use. Support for specifying a judge model is also available. For more details, refer to the [relevant parameter documentation](https://evalscope.readthedocs.io/en/latest/get_started/parameters.html).
@@ -493,7 +490,7 @@ run_task(task_cfg="config.json")
 
 ### Basic Parameter
 - `--model`: Specifies the `model_id` of the model in [ModelScope](https://modelscope.cn/), which can be automatically downloaded, e.g., [Qwen/Qwen2.5-0.5B-Instruct](https://modelscope.cn/models/Qwen/Qwen2.5-0.5B-Instruct/summary); or use the local path of the model, e.g., `/path/to/model`
-- `--datasets`: Dataset names, supports inputting multiple datasets separated by spaces. Datasets will be automatically downloaded from modelscope. For supported datasets, refer to the [Dataset List](https://evalscope.readthedocs.io/en/latest/get_started/supported_dataset.html)
+- `--datasets`: Dataset names, supports inputting multiple datasets separated by spaces. Datasets will be automatically downloaded from modelscope. For supported datasets, refer to the [Dataset List](https://evalscope.readthedocs.io/en/latest/get_started/supported_dataset/index.html)
 - `--limit`: Maximum amount of evaluation data for each dataset. If not specified, it defaults to evaluating all data. Can be used for quick validation
 
 ### Output Results
@@ -582,7 +579,7 @@ For more customized evaluations, such as customizing model parameters or dataset
 evalscope eval \
  --model Qwen/Qwen3-0.6B \
  --model-args '{"revision": "master", "precision": "torch.float16", "device_map": "auto"}' \
- --generation-config '{"do_sample":true,"temperature":0.6,"max_new_tokens":512,"chat_template_kwargs":{"enable_thinking": false}}' \
+ --generation-config '{"do_sample":true,"temperature":0.6,"max_tokens":512,"chat_template_kwargs":{"enable_thinking": false}}' \
  --dataset-args '{"gsm8k": {"few_shot_num": 0, "few_shot_random": false}}' \
  --datasets gsm8k \
  --limit 10
@@ -596,7 +593,7 @@ evalscope eval \
 - `--generation-config`: Generation parameters, passed as a JSON string and parsed as a dictionary:
   - `do_sample`: Whether to use sampling
   - `temperature`: Generation temperature
-  - `max_new_tokens`: Maximum length of generated tokens
+  - `max_tokens`: Maximum length of generated tokens
   - `chat_template_kwargs`: Model inference template parameters
 - `--dataset-args`: Settings for the evaluation dataset, passed as a JSON string where the key is the dataset name and the value is the parameters. Note that these need to correspond one-to-one with the values in the `--datasets` parameter:
   - `few_shot_num`: Number of few-shot examples