evalscope 0.7.1__py3-none-any.whl → 0.8.0__py3-none-any.whl

evalscope/utils/utils.py

@@ -5,20 +5,19 @@ import functools
 import hashlib
 import importlib
 import importlib.util
+import json
+import jsonlines as jsonl
+import numpy as np
 import os
 import random
 import re
 import sys
-from typing import Any, Dict, List, Tuple, Union
-
-import json
-import jsonlines as jsonl
-import numpy as np
 import torch
 import torch.nn.functional as F
 import yaml
+from typing import Any, Dict, List, Tuple, Union
 
-from evalscope.constants import DumpMode, OutputsStructure
+from evalscope.constants import DumpMode
 from evalscope.utils.logger import get_logger
 
 logger = get_logger()
@@ -86,13 +85,15 @@ def dump_jsonl_data(data_list, jsonl_file, dump_mode=DumpMode.OVERWRITE):
 
     jsonl_file = os.path.expanduser(jsonl_file)
 
+    if not isinstance(data_list, list):
+        data_list = [data_list]
+
     if dump_mode == DumpMode.OVERWRITE:
         dump_mode = 'w'
     elif dump_mode == DumpMode.APPEND:
         dump_mode = 'a'
     with jsonl.open(jsonl_file, mode=dump_mode) as writer:
         writer.write_all(data_list)
-    logger.info(f'Dump data to {jsonl_file} successfully.')
 
 
 def yaml_to_dict(yaml_file) -> dict:
@@ -115,7 +116,6 @@ def dict_to_yaml(d: dict, yaml_file: str):
     """
     with open(yaml_file, 'w') as f:
         yaml.dump(d, f, default_flow_style=False)
-    logger.info(f'Dump data to {yaml_file} successfully.')
 
 
 def json_to_dict(json_file) -> dict:
@@ -148,25 +148,13 @@ def get_obj_from_cfg(eval_class_ref: Any, *args, **kwargs) -> Any:
     return functools.partial(obj_cls, *args, **kwargs)
 
 
-def markdown_table(header_l, data_l):
-    md_str = f'| {" | ".join(header_l)} |'
-    md_str += f'\n| {" | ".join(["---"] * len(header_l))} |'
-    for data in data_l:
-        if isinstance(data, str):
-            data = [data]
-        assert len(data) <= len(header_l)
-        tmp = data + [''] * (len(header_l) - len(data))
-        md_str += f'\n| {" | ".join(tmp)} |'
-    return md_str
-
-
 def random_seeded_choice(seed: Union[int, str, float], choices, **kwargs):
     """Random choice with a (potentially string) seed."""
     return random.Random(seed).choices(choices, k=1, **kwargs)[0]
 
 
-def gen_hash(name: str):
-    return hashlib.md5(name.encode(encoding='UTF-8')).hexdigest()
+def gen_hash(name: str, bits: int = 32):
+    return hashlib.md5(name.encode(encoding='UTF-8')).hexdigest()[:bits]
 
 
 def dict_torch_dtype_to_str(d: Dict[str, Any]) -> dict:
@@ -313,14 +301,6 @@ class ResponseParser:
 
 
 def make_outputs_dir(root_dir: str, datasets: list, model_id: str, model_revision: str):
-    # model_revision = model_revision if model_revision is not None else 'none'
-    # now = datetime.datetime.now()
-    # format_time = now.strftime('%Y%m%d_%H%M%S')
-    # outputs_name = format_time + '_' + 'default' + '_' + model_id.replace('/', '_') + '_' + model_revision
-    # outputs_dir = os.path.join(work_dir, outputs_name)
-    # dataset_name = dataset_id.replace('/', '_')
-    # outputs_dir = os.path.join(work_dir, dataset_name)
-
     if not model_id:
         model_id = 'default'
     model_id = model_id.replace('/', '_')
@@ -328,37 +308,11 @@ def make_outputs_dir(root_dir: str, datasets: list, model_id: str, model_revisio
     if not model_revision:
         model_revision = 'default'
 
-    outputs_dir = os.path.join(root_dir, f"eval_{'-'.join(datasets)}_{model_id}_{model_revision}")
+    outputs_dir = os.path.join(root_dir, model_id, model_revision, f"eval_{'-'.join(datasets)}")
 
     return outputs_dir
 
 
-def process_outputs_structure(outputs_dir: str, is_make: bool = True) -> dict:
-    logs_dir = os.path.join(outputs_dir, 'logs')
-    predictions_dir = os.path.join(outputs_dir, 'predictions')
-    reviews_dir = os.path.join(outputs_dir, 'reviews')
-    reports_dir = os.path.join(outputs_dir, 'reports')
-    configs_dir = os.path.join(outputs_dir, 'configs')
-
-    if is_make:
-        os.makedirs(outputs_dir, exist_ok=True)
-        os.makedirs(logs_dir, exist_ok=True)
-        os.makedirs(predictions_dir, exist_ok=True)
-        os.makedirs(reviews_dir, exist_ok=True)
-        os.makedirs(reports_dir, exist_ok=True)
-        os.makedirs(configs_dir, exist_ok=True)
-
-    outputs_structure = {
-        OutputsStructure.LOGS_DIR: logs_dir,
-        OutputsStructure.PREDICTIONS_DIR: predictions_dir,
-        OutputsStructure.REVIEWS_DIR: reviews_dir,
-        OutputsStructure.REPORTS_DIR: reports_dir,
-        OutputsStructure.CONFIGS_DIR: configs_dir,
-    }
-
-    return outputs_structure
-
-
 def import_module_util(import_path_prefix: str, module_name: str, members_to_import: list) -> dict:
     """
     Import module utility function.
@@ -442,48 +396,6 @@ def split_str_parts_by(text: str, delimiters: List[str]):
     return text_list
 
 
-def calculate_loss_scale(response: str, use_loss_scale=False) -> Tuple[List[str], List[float]]:
-    """Calculate the loss scale by splitting the agent response.
-    This algorithm comes from paper: https://arxiv.org/pdf/2309.00986.pdf
-    Agent response format:
-    ```text
-        Thought: you should always think about what to do
-        Action: the action to take, should be one of the above tools[fire_recognition,
-            fire_alert, call_police, call_fireman]
-        Action Input: the input to the action
-        Observation: the result of the action
-        ... (this Thought/Action/Action Input/Observation can be repeated zero or more times)
-        Thought: I now know the final answer
-        Final Answer: the final answer to the original input question
-    ```
-    Args:
-        response: The response text
-        use_loss_scale: Use weighted loss. With this, some part of the loss will be enhanced to improve performance.
-    Returns:
-        A tuple of agent response parts and their weights.
-    """
-    if 'Action:' in response and 'Observation:' in response and use_loss_scale:
-        agent_keyword = ['Action:', 'Action Input:', 'Thought:', 'Final Answer:', 'Observation:']
-        agent_parts = split_str_parts_by(response, agent_keyword)
-        weights = []
-        agent_content = []
-        for c in agent_parts:
-            if c['key'] in ('Action:', 'Action Input:'):
-                weights += [2.0]
-                weights += [2.0]
-            elif c['key'] in ('Thought:', 'Final Answer:', ''):
-                weights += [1.0]
-                weights += [1.0]
-            elif c['key'] in ('Observation:', ):
-                weights += [2.0]
-                weights += [0.0]
-            agent_content.append(c['key'])
-            agent_content.append(c['content'])
-        return agent_content, weights
-    else:
-        return [response], [1.0]
-
-
 def get_bucket_sizes(max_length: int) -> List[int]:
     return [max_length // 4 * (i + 1) for i in range(4)]
 
@@ -504,45 +416,6 @@ def _get_closet_bucket(bucket_sizes, data_length):
     return cloest_length
 
 
-def pad_and_split_batch(padding_to, input_ids, attention_mask, labels, loss_scale, max_length, tokenizer, rank,
-                        world_size):
-    if padding_to is None:
-        longest_len = input_ids.shape[-1]
-        bucket_sizes = get_bucket_sizes(max_length)
-        bucket_data_length = _get_closet_bucket(bucket_sizes, longest_len)
-        padding_length = bucket_data_length - input_ids.shape[1]
-        input_ids = F.pad(input_ids, (0, padding_length), 'constant', tokenizer.pad_token_id)
-        attention_mask = F.pad(attention_mask, (0, padding_length), 'constant', 0)
-        if loss_scale:
-            loss_scale = F.pad(loss_scale, (0, padding_length), 'constant', 0.)
-        labels = F.pad(labels, (0, padding_length), 'constant', -100)
-
-    # manully split the batch to different DP rank.
-    batch_size = input_ids.shape[0] // world_size
-    if batch_size > 0:
-        start = rank * batch_size
-        end = (rank + 1) * batch_size
-        input_ids = input_ids[start:end, :]
-        attention_mask = attention_mask[start:end, :]
-        labels = labels[start:end, :]
-        if loss_scale:
-            loss_scale = loss_scale[start:end, :]
-    return input_ids, attention_mask, labels, loss_scale
-
-
-def get_dist_setting() -> Tuple[int, int, int, int]:
-    """return rank, local_rank, world_size, local_world_size"""
-    rank = int(os.getenv('RANK', -1))
-    local_rank = int(os.getenv('LOCAL_RANK', -1))
-    world_size = int(os.getenv('WORLD_SIZE', 1))
-    local_world_size = int(os.getenv('LOCAL_WORLD_SIZE', 1))
-    return rank, local_rank, world_size, local_world_size
-
-
-def use_torchacc() -> bool:
-    return os.getenv('USE_TORCHACC', '0') == '1'
-
-
 def is_module_installed(module_name):
     try:
         importlib.import_module(module_name)
@@ -576,6 +449,7 @@ def get_valid_list(input_list, candidate_list):
 
 def get_latest_folder_path(work_dir):
     from datetime import datetime
+
     # Get all subdirectories in the work_dir
     folders = [f for f in os.listdir(work_dir) if os.path.isdir(os.path.join(work_dir, f))]
 

evalscope/version.py

@@ -1,4 +1,4 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 
-__version__ = "0.7.1"
-__release_datetime__ = "2024-11-29 03:00:00"
+__version__ = '0.8.0'
+__release_datetime__ = '2024-12-15 00:00:00'

{evalscope-0.7.1.dist-info → evalscope-0.8.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: evalscope
-Version: 0.7.1
+Version: 0.8.0
 Summary: EvalScope: Lightweight LLMs Evaluation Framework
 Home-page: https://github.com/modelscope/evalscope
 Author: ModelScope team
@@ -29,7 +29,7 @@ Requires-Dist: nltk>=3.9
 Requires-Dist: openai
 Requires-Dist: pandas
 Requires-Dist: plotly
-Requires-Dist: pyarrow<=17.0.0
+Requires-Dist: pyarrow
 Requires-Dist: pympler
 Requires-Dist: pyyaml
 Requires-Dist: regex
@@ -62,7 +62,7 @@ Requires-Dist: nltk>=3.9; extra == "all"
 Requires-Dist: openai; extra == "all"
 Requires-Dist: pandas; extra == "all"
 Requires-Dist: plotly; extra == "all"
-Requires-Dist: pyarrow<=17.0.0; extra == "all"
+Requires-Dist: pyarrow; extra == "all"
 Requires-Dist: pympler; extra == "all"
 Requires-Dist: pyyaml; extra == "all"
 Requires-Dist: regex; extra == "all"
@@ -84,7 +84,7 @@ Requires-Dist: transformers-stream-generator; extra == "all"
 Requires-Dist: ms-opencompass>=0.1.4; extra == "all"
 Requires-Dist: ms-vlmeval>=0.0.9; extra == "all"
 Requires-Dist: mteb==1.19.4; extra == "all"
-Requires-Dist: ragas==0.2.5; extra == "all"
+Requires-Dist: ragas==0.2.7; extra == "all"
 Requires-Dist: webdataset>0.2.0; extra == "all"
 Requires-Dist: aiohttp; extra == "all"
 Requires-Dist: fastapi; extra == "all"
@@ -129,48 +129,52 @@ Requires-Dist: transformers; extra == "perf"
 Requires-Dist: unicorn; extra == "perf"
 Provides-Extra: rag
 Requires-Dist: mteb==1.19.4; extra == "rag"
-Requires-Dist: ragas==0.2.5; extra == "rag"
+Requires-Dist: ragas==0.2.7; extra == "rag"
 Requires-Dist: webdataset>0.2.0; extra == "rag"
 Provides-Extra: vlmeval
 Requires-Dist: ms-vlmeval>=0.0.9; extra == "vlmeval"
 
+<p align="center">
+    <br>
+    <img src="docs/en/_static/images/evalscope_logo.png"/>
+    <br>
+<p>
 
 
-![](docs/en/_static/images/evalscope_logo.png)
-
 <p align="center">
-    English | <a href="README_zh.md">简体中文</a>
+  <a href="README_zh.md">中文</a> &nbsp ｜ &nbsp English &nbsp
 </p>
 
 <p align="center">
-  <a href="https://badge.fury.io/py/evalscope"><img src="https://badge.fury.io/py/evalscope.svg" alt="PyPI version" height="18"></a>
-  <a href="https://pypi.org/project/evalscope"><img alt="PyPI - Downloads" src="https://static.pepy.tech/badge/evalscope">
-  </a>
-  <a href="https://github.com/modelscope/evalscope/pulls"><img src="https://img.shields.io/badge/PR-welcome-55EB99.svg"></a>
-  <a href='https://evalscope.readthedocs.io/en/latest/?badge=latest'>
-      <img src='https://readthedocs.org/projects/evalscope-en/badge/?version=latest' alt='Documentation Status' />
-  </a>
-  <br>
-  <a href="https://evalscope.readthedocs.io/en/latest/">📖 Documents</a>
+<img src="https://img.shields.io/badge/python-%E2%89%A53.8-5be.svg">
+<a href="https://badge.fury.io/py/evalscope"><img src="https://badge.fury.io/py/evalscope.svg" alt="PyPI version" height="18"></a>
+<a href="https://pypi.org/project/evalscope"><img alt="PyPI - Downloads" src="https://static.pepy.tech/badge/evalscope"></a>
+<a href="https://github.com/modelscope/evalscope/pulls"><img src="https://img.shields.io/badge/PR-welcome-55EB99.svg"></a>
+<a href='https://evalscope.readthedocs.io/en/latest/?badge=latest'><img src='https://readthedocs.org/projects/evalscope/badge/?version=latest' alt='Documentation Status' /></a>
+<p>
+
+<p align="center">
+<a href="https://evalscope.readthedocs.io/zh-cn/latest/"> 📖  中文文档</a> &nbsp ｜ &nbsp <a href="https://evalscope.readthedocs.io/en/latest/"> 📖  English Documents</a>
 <p>
 
 > ⭐ If you like this project, please click the "Star" button at the top right to support us. Your support is our motivation to keep going!
 
-## 📋 Table of Contents
+## 📋 Contents
 - [Introduction](#introduction)
 - [News](#News)
 - [Installation](#installation)
 - [Quick Start](#quick-start)
 - [Evaluation Backend](#evaluation-backend)
 - [Custom Dataset Evaluation](#custom-dataset-evaluation)
-- [Offline Evaluation](#offline-evaluation)
-- [Arena Mode](#arena-mode)
 - [Model Serving Performance Evaluation](#Model-Serving-Performance-Evaluation)
+- [Arena Mode](#arena-mode)
 
 
 ## 📝 Introduction
 
-EvalScope is the official model evaluation and performance benchmarking framework launched by the [ModelScope](https://modelscope.cn/) community. It comes with built-in common benchmarks and evaluation metrics, such as MMLU, CMMLU, C-Eval, GSM8K, ARC, HellaSwag, TruthfulQA, MATH, and HumanEval. EvalScope supports various types of model evaluations, including LLMs, multimodal LLMs, embedding models, and reranker models. It is also applicable to multiple evaluation scenarios, such as end-to-end RAG evaluation, arena mode, and model inference performance stress testing. Moreover, with the seamless integration of the ms-swift training framework, evaluations can be initiated with a single click, providing full end-to-end support from model training to evaluation 🚀
+EvalScope is [ModelScope](https://modelscope.cn/)'s official framework for model evaluation and benchmarking, designed for diverse assessment needs. It supports various model types including large language models, multimodal, embedding, reranker, and CLIP models.
+
+The framework accommodates multiple evaluation scenarios such as end-to-end RAG evaluation, arena mode, and inference performance testing. It features built-in benchmarks and metrics like MMLU, CMMLU, C-Eval, and GSM8K. Seamlessly integrated with the [ms-swift](https://github.com/modelscope/ms-swift) training framework, EvalScope enables one-click evaluations, offering comprehensive support for model training and assessment 🚀
 
 <p align="center">
   <img src="docs/en/_static/images/evalscope_framework.png" width="70%">
@@ -192,6 +196,7 @@ The architecture includes the following modules:
 
 
 ## 🎉 News
+- 🔥 **[2024.12.13]** Model evaluation optimization: no need to pass the `--template-type` parameter anymore; supports starting evaluation with `evalscope eval --args`. Refer to the [📖 User Guide](https://evalscope.readthedocs.io/en/latest/get_started/basic_usage.html) for more details.
 - 🔥 **[2024.11.26]** The model inference service performance evaluator has been completely refactored: it now supports local inference service startup and Speed Benchmark; asynchronous call error handling has been optimized. For more details, refer to the [📖 User Guide](https://evalscope.readthedocs.io/en/latest/user_guides/stress_test/index.html).
 - 🔥 **[2024.10.31]** The best practice for evaluating Multimodal-RAG has been updated, please check the [📖 Blog](https://evalscope.readthedocs.io/zh-cn/latest/blog/RAG/multimodal_RAG.html#multimodal-rag) for more details.
 - 🔥 **[2024.10.23]** Supports multimodal RAG evaluation, including the assessment of image-text retrieval using [CLIP_Benchmark](https://evalscope.readthedocs.io/en/latest/user_guides/backend/rageval_backend/clip_benchmark.html), and extends [RAGAS](https://evalscope.readthedocs.io/en/latest/user_guides/backend/rageval_backend/ragas.html) to support end-to-end multimodal metrics evaluation.
@@ -263,124 +268,129 @@ We recommend using conda to manage your environment and installing dependencies
 
 ## 🚀 Quick Start
 
-### 1. Simple Evaluation
-To evaluate a model using default settings on specified datasets, follow the process below:
+To evaluate a model on specified datasets using default configurations, this framework supports two ways to initiate evaluation tasks: using the command line or using Python code.
 
-#### Installation using pip
+### Method 1. Using Command Line
 
-You can execute this in any directory:
+Execute the `eval` command in any directory:
 ```bash
-python -m evalscope.run \
+evalscope eval \
  --model Qwen/Qwen2.5-0.5B-Instruct \
- --template-type qwen \
- --datasets gsm8k ceval \
- --limit 10
+ --datasets gsm8k arc \
+ --limit 5
 ```
 
-#### Installation from source
+### Method 2. Using Python Code
 
-You need to execute this in the `evalscope` directory:
-```bash
-python evalscope/run.py \
- --model Qwen/Qwen2.5-0.5B-Instruct \
- --template-type qwen \
- --datasets gsm8k ceval \
- --limit 10
-```
+When using Python code for evaluation, you need to submit the evaluation task using the `run_task` function, passing a `TaskConfig` as a parameter. It can also be a Python dictionary, yaml file path, or json file path, for example:
 
-> If prompted with `Do you wish to run the custom code? [y/N]`, please type `y`.
+**Using Python Dictionary**
 
-**Results (tested with only 10 samples)**
-```text
-Report table:
-+-----------------------+--------------------+-----------------+
-| Model                 | ceval              | gsm8k           |
-+=======================+====================+=================+
-| Qwen2.5-0.5B-Instruct | (ceval/acc) 0.5577 | (gsm8k/acc) 0.5 |
-+-----------------------+--------------------+-----------------+
+```python
+from evalscope.run import run_task
+
+task_cfg = {
+    'model': 'Qwen/Qwen2.5-0.5B-Instruct',
+    'datasets': ['gsm8k', 'arc'],
+    'limit': 5
+}
+
+run_task(task_cfg=task_cfg)
 ```
 
+<details><summary>More Startup Methods</summary>
 
-#### Basic Parameter Descriptions
-- `--model`: Specifies the `model_id` of the model on [ModelScope](https://modelscope.cn/), allowing automatic download. For example, see the [Qwen2-0.5B-Instruct model link](https://modelscope.cn/models/qwen/Qwen2-0.5B-Instruct/summary); you can also use a local path, such as `/path/to/model`.
-- `--template-type`: Specifies the template type corresponding to the model. Refer to the `Default Template` field in the [template table](https://swift.readthedocs.io/en/latest/Instruction/Supported-models-datasets.html#llm) for filling in this field.
-- `--datasets`: The dataset name, allowing multiple datasets to be specified, separated by spaces; these datasets will be automatically downloaded. Refer to the [supported datasets list](https://evalscope.readthedocs.io/en/latest/get_started/supported_dataset.html) for available options.
-- `--limit`: Maximum number of evaluation samples per dataset; if not specified, all will be evaluated, which is useful for quick validation.
+**Using `TaskConfig`**
 
+```python
+from evalscope.run import run_task
+from evalscope.config import TaskConfig
 
-### 2. Parameterized Evaluation
-If you wish to conduct a more customized evaluation, such as modifying model parameters or dataset parameters, you can use the following commands:
+task_cfg = TaskConfig(
+    model='Qwen/Qwen2.5-0.5B-Instruct',
+    datasets=['gsm8k', 'arc'],
+    limit=5
+)
 
-**Example 1:**
-```shell
-python evalscope/run.py \
- --model qwen/Qwen2-0.5B-Instruct \
- --template-type qwen \
- --model-args revision=master,precision=torch.float16,device_map=auto \
- --datasets gsm8k ceval \
- --use-cache true \
- --limit 10
+run_task(task_cfg=task_cfg)
 ```
 
-**Example 2:**
-```shell
-python evalscope/run.py \
- --model qwen/Qwen2-0.5B-Instruct \
- --template-type qwen \
- --generation-config do_sample=false,temperature=0.0 \
- --datasets ceval \
- --dataset-args '{"ceval": {"few_shot_num": 0, "few_shot_random": false}}' \
- --limit 10
+**Using `yaml` file**
+
+`config.yaml`:
+```yaml
+model: Qwen/Qwen2.5-0.5B-Instruct
+datasets:
+  - gsm8k
+  - arc
+limit: 5
 ```
 
-#### Parameter Descriptions
-In addition to the three [basic parameters](#basic-parameter-descriptions), the other parameters are as follows:
-- `--model-args`: Model loading parameters, separated by commas, in `key=value` format.
-- `--generation-config`: Generation parameters, separated by commas, in `key=value` format.
-  - `do_sample`: Whether to use sampling, default is `false`.
-  - `max_new_tokens`: Maximum generation length, default is 1024.
-  - `temperature`: Sampling temperature.
-  - `top_p`: Sampling threshold.
-  - `top_k`: Sampling threshold.
-- `--use-cache`: Whether to use local cache, default is `false`. If set to `true`, previously evaluated model and dataset combinations will not be evaluated again, and will be read directly from the local cache.
-- `--dataset-args`: Evaluation dataset configuration parameters, provided in JSON format, where the key is the dataset name and the value is the parameter; note that these must correspond one-to-one with the values in `--datasets`.
-  - `--few_shot_num`: Number of few-shot examples.
-  - `--few_shot_random`: Whether to randomly sample few-shot data; if not specified, defaults to `true`.
-
-
-### 3. Use the run_task Function to Submit an Evaluation Task
-Using the `run_task` function to submit an evaluation task requires the same parameters as the command line. You need to pass a dictionary as the parameter, which includes the following fields:
-
-#### 1. Configuration Task Dictionary Parameters
 ```python
-import torch
-from evalscope.constants import DEFAULT_ROOT_CACHE_DIR
-
-# Example
-your_task_cfg = {
-        'model_args': {'revision': None, 'precision': torch.float16, 'device_map': 'auto'},
-        'generation_config': {'do_sample': False, 'repetition_penalty': 1.0, 'max_new_tokens': 512},
-        'dataset_args': {},
-        'dry_run': False,
-        'model': 'qwen/Qwen2-0.5B-Instruct',
-        'template_type': 'qwen',
-        'datasets': ['arc', 'hellaswag'],
-        'work_dir': DEFAULT_ROOT_CACHE_DIR,
-        'outputs': DEFAULT_ROOT_CACHE_DIR,
-        'mem_cache': False,
-        'dataset_hub': 'ModelScope',
-        'dataset_dir': DEFAULT_ROOT_CACHE_DIR,
-        'limit': 10,
-        'debug': False
-    }
+from evalscope.run import run_task
+
+run_task(task_cfg="config.yaml")
+```
+
+**Using `json` file**
+
+`config.json`:
+```json
+{
+    "model": "Qwen/Qwen2.5-0.5B-Instruct",
+    "datasets": ["gsm8k", "arc"],
+    "limit": 5
+}
 ```
-Here, `DEFAULT_ROOT_CACHE_DIR` is set to `'~/.cache/evalscope'`.
 
-#### 2. Execute Task with run_task
 ```python
 from evalscope.run import run_task
-run_task(task_cfg=your_task_cfg)
+
+run_task(task_cfg="config.json")
+```
+</details>
+
+### Basic Parameter
+- `--model`: Specifies the `model_id` of the model in [ModelScope](https://modelscope.cn/), which can be automatically downloaded, e.g., [Qwen/Qwen2.5-0.5B-Instruct](https://modelscope.cn/models/Qwen/Qwen2.5-0.5B-Instruct/summary); or use the local path of the model, e.g., `/path/to/model`
+- `--datasets`: Dataset names, supports inputting multiple datasets separated by spaces. Datasets will be automatically downloaded from modelscope. For supported datasets, refer to the [Dataset List](https://evalscope.readthedocs.io/en/latest/get_started/supported_dataset.html)
+- `--limit`: Maximum amount of evaluation data for each dataset. If not specified, it defaults to evaluating all data. Can be used for quick validation
+
+### Output Results
 ```
++-----------------------+-------------------+-----------------+
+| Model                 | ai2_arc           | gsm8k           |
++=======================+===================+=================+
+| Qwen2.5-0.5B-Instruct | (ai2_arc/acc) 0.6 | (gsm8k/acc) 0.6 |
++-----------------------+-------------------+-----------------+
+```
+
+## ⚙️ Complex Evaluation
+For more customized evaluations, such as customizing model parameters or dataset parameters, you can use the following command. The evaluation startup method is the same as simple evaluation. Below shows how to start the evaluation using the `eval` command:
+
+```shell
+evalscope eval \
+ --model Qwen/Qwen2.5-0.5B-Instruct \
+ --model-args revision=master,precision=torch.float16,device_map=auto \
+ --generation-config do_sample=true,temperature=0.5 \
+ --dataset-args '{"gsm8k": {"few_shot_num": 0, "few_shot_random": false}}' \
+ --datasets gsm8k \
+ --limit 10
+```
+
+### Parameter
+- `--model-args`: Model loading parameters, separated by commas in `key=value` format. Default parameters:
+  - `revision`: Model version, default is `master`
+  - `precision`: Model precision, default is `auto`
+  - `device_map`: Model device allocation, default is `auto`
+- `--generation-config`: Generation parameters, separated by commas in `key=value` format. Default parameters:
+  - `do_sample`: Whether to use sampling, default is `false`
+  - `max_length`: Maximum length, default is 2048
+  - `max_new_tokens`: Maximum length of generation, default is 512
+- `--dataset-args`: Configuration parameters for evaluation datasets, passed in `json` format. The key is the dataset name, and the value is the parameters. Note that it needs to correspond one-to-one with the values in the `--datasets` parameter:
+  - `few_shot_num`: Number of few-shot examples
+  - `few_shot_random`: Whether to randomly sample few-shot data, if not set, defaults to `true`
+
+Reference: [Full Parameter Description](https://evalscope.readthedocs.io/en/latest/get_started/parameters.html)
 
 
 ## Evaluation Backend
@@ -418,12 +428,7 @@ Speed Benchmark Results:
 ```
 
 ## Custom Dataset Evaluation
-EvalScope supports custom dataset evaluation. For detailed information, please refer to the Custom Dataset Evaluation [📖User Guide](https://evalscope.readthedocs.io/en/latest/advanced_guides/custom_dataset.html)
-
-## Offline Evaluation
-You can use local dataset to evaluate the model without internet connection.
-
-Refer to: Offline Evaluation [📖 User Guide](https://evalscope.readthedocs.io/en/latest/user_guides/offline_evaluation.html)
+EvalScope supports custom dataset evaluation. For detailed information, please refer to the Custom Dataset Evaluation [📖User Guide](https://evalscope.readthedocs.io/en/latest/advanced_guides/custom_dataset/index.html)
 
 
 ## Arena Mode