evalscope 0.12.1__py3-none-any.whl → 0.13.1__py3-none-any.whl

evalscope/perf/benchmark.py

@@ -150,39 +150,45 @@ async def statistic_benchmark_metric_worker(benchmark_data_queue: asyncio.Queue,
         name = args.name if args.name else f'{args.model_id}_{current_time}'
         wandb.init(project='perf_benchmark', name=name, config=args.to_dict())
 
-    with sqlite3.connect(result_db_path) as con:
-        cursor = con.cursor()
-        create_result_table(cursor)
-        with tqdm(desc='Processing') as pbar:
-            while not (data_process_completed_event.is_set() and benchmark_data_queue.empty()):
-                try:
-                    # Attempt to get benchmark data from the queue with a timeout
-                    benchmark_data = await asyncio.wait_for(benchmark_data_queue.get(), timeout=0.01)
-                    benchmark_data_queue.task_done()
-                except asyncio.TimeoutError:
-                    # If timeout, continue to the next iteration
-                    continue
+    collected_benchmark_data = []
 
-                # Update metrics based on the benchmark data
-                metrics.update_metrics(benchmark_data, api_plugin)
+    with tqdm(desc='Processing') as pbar:
+        while not (data_process_completed_event.is_set() and benchmark_data_queue.empty()):
+            try:
+                # Attempt to get benchmark data from the queue with a timeout
+                benchmark_data = await asyncio.wait_for(benchmark_data_queue.get(), timeout=0.01)
+                benchmark_data_queue.task_done()
+            except asyncio.TimeoutError:
+                # If timeout, continue to the next iteration
+                continue
 
-                # Insert benchmark data into the database and commit the transaction
-                insert_benchmark_data(cursor, benchmark_data)
-                con.commit()
+            # Update metrics based on the benchmark data
+            metrics.update_metrics(benchmark_data, api_plugin)
 
-                # Create a message with the updated metrics
-                message = metrics.create_message()
+            # Collect benchmark data for later database insertion
+            collected_benchmark_data.append(benchmark_data)
 
-                # Log the message to wandb if the api key is provided
-                if args.wandb_api_key:
-                    wandb.log(message)
+            # Create a message with the updated metrics
+            message = metrics.create_message()
 
-                # Log the message to the logger every n queries
-                if int(metrics.n_total_queries) % args.log_every_n_query == 0:
-                    msg = json.dumps(message, ensure_ascii=False, indent=2)
-                    logger.info(msg)
+            # Log the message to wandb if the api key is provided
+            if args.wandb_api_key:
+                wandb.log(message)
 
-                pbar.update(1)  # Update the progress bar
+            # Log the message to the logger every n queries
+            if int(metrics.n_total_queries) % args.log_every_n_query == 0:
+                msg = json.dumps(message, ensure_ascii=False, indent=2)
+                logger.info(msg)
+
+            pbar.update(1)  # Update the progress bar
+
+    # Now perform database operations after all benchmark data has been processed
+    with sqlite3.connect(result_db_path) as con:
+        cursor = con.cursor()
+        create_result_table(cursor)
+        for benchmark_data in collected_benchmark_data:
+            insert_benchmark_data(cursor, benchmark_data)
+        con.commit()
 
     return metrics, result_db_path
 
@@ -199,7 +205,7 @@ async def start_server(args: Arguments) -> bool:
         else:
             args.url = f'http://127.0.0.1:{args.port}/v1/chat/completions'
 
-    if not await test_connection(args):
+    if (not args.no_test_connection) and (not await test_connection(args)):
         raise TimeoutError('Test connection failed')
 
 
@@ -209,9 +215,14 @@ async def benchmark(args: Arguments) -> None:
         loop = asyncio.get_running_loop()
         add_signal_handlers(loop)
 
+    # init queue
     request_queue = asyncio.Queue()
     benchmark_data_queue = asyncio.Queue()
 
+    # reset event
+    query_send_completed_event.clear()
+    data_process_completed_event.clear()
+
     async def create_send_request_tasks():
         tasks: List[asyncio.Task] = []
         for idx in range(args.parallel):

evalscope/perf/http_client.py

@@ -145,7 +145,15 @@ async def test_connection(args: Arguments) -> bool:
         client = AioHttpClient(args)
         async with client:
             if 'chat/completions' in args.url:
-                request = {'messages': [{'role': 'user', 'content': 'hello'}], 'model': args.model, 'max_tokens': 10}
+                request = {
+                    'messages': [{
+                        'role': 'user',
+                        'content': 'hello'
+                    }],
+                    'model': args.model,
+                    'max_tokens': 10,
+                    'stream': args.stream
+                }
             else:
                 request = {'prompt': 'hello', 'model': args.model, 'max_tokens': 10}
             async for is_error, state_code, response_data in client.post(request):

evalscope/perf/main.py

@@ -32,9 +32,10 @@ def run_perf_benchmark(args):
     if platform.system() == 'Windows':
         asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
 
-    loop = asyncio.get_event_loop()
+    loop = asyncio.new_event_loop()
     if platform.system() != 'Windows':
         add_signal_handlers(loop)
+
     loop.run_until_complete(benchmark(args))
 
 

evalscope/perf/plugin/datasets/__init__.py

@@ -3,4 +3,5 @@ from evalscope.perf.plugin.datasets.flickr8k import FlickrDatasetPlugin
 from evalscope.perf.plugin.datasets.line_by_line import LineByLineDatasetPlugin
 from evalscope.perf.plugin.datasets.longalpaca import LongAlpacaDatasetPlugin
 from evalscope.perf.plugin.datasets.openqa import OpenqaDatasetPlugin
+from evalscope.perf.plugin.datasets.random_dataset import RandomDatasetPlugin
 from evalscope.perf.plugin.datasets.speed_benchmark import SpeedBenchmarkDatasetPlugin, SpeedBenchmarkLongDatasetPlugin

evalscope/perf/plugin/datasets/openqa.py

@@ -1,5 +1,5 @@
 import json
-import subprocess
+import os
 from typing import Any, Dict, Iterator, List
 
 from evalscope.perf.arguments import Arguments
@@ -18,16 +18,11 @@ class OpenqaDatasetPlugin(DatasetPluginBase):
 
     def build_messages(self) -> Iterator[List[Dict]]:
         if not self.query_parameters.dataset_path:
-            subprocess.call([
-                'modelscope',
-                'download',
-                '--dataset',
-                'AI-ModelScope/HC3-Chinese',
-                'open_qa.jsonl',
-                '--local_dir',
-                './data',
-            ])
-            self.query_parameters.dataset_path = './data/open_qa.jsonl'
+            from modelscope import dataset_snapshot_download
+
+            file_name = 'open_qa.jsonl'
+            local_path = dataset_snapshot_download('AI-ModelScope/HC3-Chinese', allow_patterns=[file_name])
+            self.query_parameters.dataset_path = os.path.join(local_path, file_name)
 
         for item in self.dataset_line_by_line(self.query_parameters.dataset_path):
             item = json.loads(item)

evalscope/perf/plugin/datasets/random_dataset.py

@@ -0,0 +1,51 @@
+import numpy as np
+from typing import Dict, Iterator, List
+
+from evalscope.perf.arguments import Arguments
+from evalscope.perf.plugin.datasets.base import DatasetPluginBase
+from evalscope.perf.plugin.registry import register_dataset
+
+
+@register_dataset('random')
+class RandomDatasetPlugin(DatasetPluginBase):
+    """Read dataset and return prompt.
+    """
+
+    def __init__(self, query_parameters: Arguments):
+        super().__init__(query_parameters)
+        assert self.query_parameters.tokenizer_path, 'Tokenizer path is required for random data generation, please provide it with `--tokenizer_path`.'  # noqa: E501
+
+        from modelscope import AutoTokenizer
+        self.tokenizer = AutoTokenizer.from_pretrained(self.query_parameters.tokenizer_path, trust_remote_code=True)
+        self.prefix_length = self.query_parameters.prefix_length
+        self.prefix_ids = self.get_random_inputs(self.prefix_length)
+        self.template_len = self.get_template_len()
+        self.number = self.query_parameters.number or 1
+
+    def build_messages(self) -> Iterator[List[Dict]]:
+        min_prompt_length = self.query_parameters.min_prompt_length - self.template_len
+        max_prompt_length = self.query_parameters.max_prompt_length - self.template_len + 1
+
+        assert min_prompt_length >= 0, f'min_prompt_length should be greater than or equal to the template length {self.template_len}.'  # noqa: E501
+        assert max_prompt_length >= min_prompt_length, 'max_prompt_length should be greater than or equal to min_prompt_length.'  # noqa: E501
+
+        # refer to https://github.com/vllm-project/vllm/blob/ed6e9075d31e32c8548b480a47d1ffb77da1f54c/benchmarks/benchmark_serving.py#L366C1-L399C1  # noqa: E501
+        input_lens = np.random.randint(min_prompt_length, max_prompt_length, size=self.number)
+        offsets = np.random.randint(0, self.tokenizer.vocab_size, size=self.number)
+
+        for i in range(self.number):
+            prompt_ids = (offsets[i] + i + np.arange(input_lens[i])) % self.tokenizer.vocab_size
+            prompt = self.tokenizer.decode(
+                self.prefix_ids + prompt_ids.tolist(), skip_special_tokens=False, clean_up_tokenization_spaces=False)
+            yield [{'role': 'user', 'content': prompt}]
+
+    def get_random_inputs(self, length: int) -> List[int]:
+        if length <= 0:
+            return []
+        input_ids = np.random.randint(0, self.tokenizer.vocab_size, size=length).tolist()
+        return input_ids
+
+    def get_template_len(self):
+        empty_message = [{'role': 'user', 'content': ''}]
+        template = self.tokenizer.apply_chat_template(empty_message, tokenize=True, add_generation_prompt=True)
+        return len(template)

evalscope/perf/utils/db_util.py

@@ -2,6 +2,7 @@ import base64
 import json
 import os
 import pickle
+import re
 import sqlite3
 import sys
 from datetime import datetime
@@ -91,6 +92,8 @@ def insert_benchmark_data(cursor: sqlite3.Cursor, benchmark_data: BenchmarkData)
 def get_output_path(args: Arguments) -> str:
     current_time = datetime.now().strftime('%Y%m%d_%H%M%S')
     output_path = os.path.join(args.outputs_dir, current_time, f'{args.name or args.model_id}')
+    # Filter illegal characters
+    output_path = re.sub(r'[<>:"|?*]', '_', output_path)
     if not os.path.exists(output_path):
         os.makedirs(output_path, exist_ok=True)
     logger.info(f'Save the result to: {output_path}')

evalscope/run.py

@@ -39,9 +39,11 @@ def run_single_task(task_cfg: TaskConfig, run_time: str) -> dict:
     configure_logging(task_cfg.debug, os.path.join(outputs.logs_dir, 'eval_log.log'))
 
     if task_cfg.eval_backend != EvalBackend.NATIVE:
-        return run_non_native_backend(task_cfg, outputs)
+        result = run_non_native_backend(task_cfg, outputs)
     else:
-        return evaluate_model(task_cfg, outputs)
+        result = evaluate_model(task_cfg, outputs)
+
+    return result
 
 
 def setup_work_directory(task_cfg: TaskConfig, run_time: str):
@@ -117,6 +119,16 @@ def evaluate_model(task_cfg: TaskConfig, outputs: OutputsStructure) -> dict:
         res_dict = evaluator.eval()
         eval_results[evaluator.dataset_name] = res_dict
 
+    # Clean up
+    if base_model is not None:
+        import gc
+        import torch
+
+        del base_model
+        del evaluators
+        torch.cuda.empty_cache()
+        gc.collect()
+
     return eval_results
 
 
@@ -132,7 +144,7 @@ def create_evaluator(task_cfg: TaskConfig, dataset_name: str, outputs: OutputsSt
         # EvaluatorCollection is a collection of evaluators
         from evalscope.collections import EvaluatorCollection
         data_adapter = benchmark.get_data_adapter(config=task_cfg.dataset_args.get(dataset_name, {}))
-        return EvaluatorCollection(task_cfg, data_adapter, outputs)
+        return EvaluatorCollection(task_cfg, data_adapter, outputs, base_model)
 
     # Initialize model adapter
     model_adapter = initialize_model_adapter(task_cfg, benchmark, base_model)

evalscope/third_party/longbench_write/infer.py

@@ -8,7 +8,7 @@ import random
 import torch
 from typing import List
 
-from evalscope.models.api import OpenaiApi
+from evalscope.third_party.longbench_write.tools.openai_api import OpenaiApi
 from evalscope.third_party.longbench_write.utils import count_words
 from evalscope.utils import get_logger
 

evalscope/version.py

@@ -1,4 +1,4 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 
-__version__ = '0.12.1'
-__release_datetime__ = '2025-03-10 21:00:00'
+__version__ = '0.13.1'
+__release_datetime__ = '2025-03-24 18:00:00'

{evalscope-0.12.1.dist-info → evalscope-0.13.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: evalscope
-Version: 0.12.1
+Version: 0.13.1
 Summary: EvalScope: Lightweight LLMs Evaluation Framework
 Home-page: https://github.com/modelscope/evalscope
 Author: ModelScope team
@@ -175,16 +175,29 @@ Requires-Dist: ms-vlmeval>=0.0.9; extra == "vlmeval"
 > ⭐ If you like this project, please click the "Star" button at the top right to support us. Your support is our motivation to keep going!
 
 ## 📋 Contents
-- [Introduction](#-introduction)
-- [News](#-news)
-- [Installation](#️-installation)
-- [Quick Start](#-quick-start)
+- [📋 Contents](#-contents)
+- [📝 Introduction](#-introduction)
+- [☎ User Groups](#-user-groups)
+- [🎉 News](#-news)
+- [🛠️ Installation](#️-installation)
+  - [Method 1: Install Using pip](#method-1-install-using-pip)
+  - [Method 2: Install from Source](#method-2-install-from-source)
+- [🚀 Quick Start](#-quick-start)
+  - [Method 1. Using Command Line](#method-1-using-command-line)
+  - [Method 2. Using Python Code](#method-2-using-python-code)
+  - [Basic Parameter](#basic-parameter)
+  - [Output Results](#output-results)
+- [📈 Visualization of Evaluation Results](#-visualization-of-evaluation-results)
+- [🌐 Evaluation of Specified Model API](#-evaluation-of-specified-model-api)
+- [⚙️ Custom Parameter Evaluation](#️-custom-parameter-evaluation)
+  - [Parameter](#parameter)
 - [Evaluation Backend](#evaluation-backend)
-- [Custom Dataset Evaluation](#️-custom-dataset-evaluation)
-- [Model Serving Performance Evaluation](#-model-serving-performance-evaluation)
-- [Arena Mode](#-arena-mode)
-- [Contribution](#️-contribution)
-- [Roadmap](#-roadmap)
+- [📈 Model Serving Performance Evaluation](#-model-serving-performance-evaluation)
+- [🖊️ Custom Dataset Evaluation](#️-custom-dataset-evaluation)
+- [🏟️ Arena Mode](#️-arena-mode)
+- [👷‍♂️ Contribution](#️-contribution)
+- [🔜 Roadmap](#-roadmap)
+- [Star History](#star-history)
 
 
 ## 📝 Introduction
@@ -226,6 +239,9 @@ Please scan the QR code below to join our community groups:
 
 ## 🎉 News
 
+- 🔥 **[2025.03.20]** The model inference service stress testing now supports generating prompts of specified length using random values. Refer to the [user guide](https://evalscope.readthedocs.io/en/latest/user_guides/stress_test/examples.html#using-the-random-dataset) for more details.
+- 🔥 **[2025.03.13]** Added support for the [LiveCodeBench](https://www.modelscope.cn/datasets/AI-ModelScope/code_generation_lite/summary) code evaluation benchmark, which can be used by specifying `live_code_bench`. Supports evaluating QwQ-32B on LiveCodeBench, refer to the [best practices](https://evalscope.readthedocs.io/en/latest/best_practice/eval_qwq.html).
+- 🔥 **[2025.03.11]** Added support for the [SimpleQA](https://modelscope.cn/datasets/AI-ModelScope/SimpleQA/summary) and [Chinese SimpleQA](https://modelscope.cn/datasets/AI-ModelScope/Chinese-SimpleQA/summary) evaluation benchmarks. These are used to assess the factual accuracy of models, and you can specify `simple_qa` and `chinese_simpleqa` for use. Support for specifying a judge model is also available. For more details, refer to the [relevant parameter documentation](https://evalscope.readthedocs.io/en/latest/get_started/parameters.html).
 - 🔥 **[2025.03.07]** Added support for the [QwQ-32B](https://modelscope.cn/models/Qwen/QwQ-32B/summary) model, evaluate the model's reasoning ability and reasoning efficiency, refer to [📖 Best Practices for QwQ-32B Evaluation](https://evalscope.readthedocs.io/en/latest/best_practice/eval_qwq.html) for more details.
 - 🔥 **[2025.03.04]** Added support for the [SuperGPQA](https://modelscope.cn/datasets/m-a-p/SuperGPQA/summary) dataset, which covers 13 categories, 72 first-level disciplines, and 285 second-level disciplines, totaling 26,529 questions. You can use it by specifying `super_gpqa`.
 - 🔥 **[2025.03.03]** Added support for evaluating the IQ and EQ of models. Refer to [📖 Best Practices for IQ and EQ Evaluation](https://evalscope.readthedocs.io/en/latest/best_practice/iquiz.html) to find out how smart your AI is!
@@ -262,23 +278,24 @@ Please scan the QR code below to join our community groups:
 We recommend using conda to manage your environment and installing dependencies with pip:
 
 1. Create a conda environment (optional)
-   ```shell
-   # It is recommended to use Python 3.10
-   conda create -n evalscope python=3.10
-   # Activate the conda environment
-   conda activate evalscope
-   ```
+    ```shell
+    # It is recommended to use Python 3.10
+    conda create -n evalscope python=3.10
+    # Activate the conda environment
+    conda activate evalscope
+    ```
 
 2. Install dependencies using pip
-   ```shell
-   pip install evalscope                # Install Native backend (default)
-   # Additional options
-   pip install evalscope[opencompass]   # Install OpenCompass backend
-   pip install evalscope[vlmeval]       # Install VLMEvalKit backend
-   pip install evalscope[rag]           # Install RAGEval backend
-   pip install evalscope[perf]          # Install Perf dependencies
-   pip install evalscope[all]           # Install all backends (Native, OpenCompass, VLMEvalKit, RAGEval)
-   ```
+    ```shell
+    pip install evalscope                # Install Native backend (default)
+    # Additional options
+    pip install 'evalscope[opencompass]'   # Install OpenCompass backend
+    pip install 'evalscope[vlmeval]'       # Install VLMEvalKit backend
+    pip install 'evalscope[rag]'           # Install RAGEval backend
+    pip install 'evalscope[perf]'          # Install dependencies for the model performance testing module
+    pip install 'evalscope[app]'           # Install dependencies for visualization
+    pip install 'evalscope[all]'           # Install all backends (Native, OpenCompass, VLMEvalKit, RAGEval)
+    ```
 
 > [!WARNING]
 > As the project has been renamed to `evalscope`, for versions `v0.4.3` or earlier, you can install using the following command:
@@ -292,21 +309,22 @@ We recommend using conda to manage your environment and installing dependencies
 
 ### Method 2: Install from Source
 1. Download the source code
-   ```shell
-   git clone https://github.com/modelscope/evalscope.git
-   ```
+    ```shell
+    git clone https://github.com/modelscope/evalscope.git
+    ```
 
 2. Install dependencies
-   ```shell
-   cd evalscope/
-   pip install -e .                  # Install Native backend
-   # Additional options
-   pip install -e '.[opencompass]'   # Install OpenCompass backend
-   pip install -e '.[vlmeval]'       # Install VLMEvalKit backend
-   pip install -e '.[rag]'           # Install RAGEval backend
-   pip install -e '.[perf]'          # Install Perf dependencies
-   pip install -e '.[all]'           # Install all backends (Native, OpenCompass, VLMEvalKit, RAGEval)
-   ```
+    ```shell
+    cd evalscope/
+    pip install -e .                  # Install Native backend
+    # Additional options
+    pip install -e '.[opencompass]'   # Install OpenCompass backend
+    pip install -e '.[vlmeval]'       # Install VLMEvalKit backend
+    pip install -e '.[rag]'           # Install RAGEval backend
+    pip install -e '.[perf]'          # Install Perf dependencies
+    pip install -e '.[app]'           # Install visualization dependencies
+    pip install -e '.[all]'           # Install all backends (Native, OpenCompass, VLMEvalKit, RAGEval)
+    ```
 
 
 ## 🚀 Quick Start