evalscope 0.13.0__py3-none-any.whl → 0.13.2__py3-none-any.whl

evalscope/perf/benchmark.py

@@ -9,7 +9,7 @@ import threading
 import time
 from http import HTTPStatus
 from tqdm import tqdm
-from typing import List
+from typing import AsyncGenerator, List
 
 from evalscope.perf.arguments import Arguments
 from evalscope.perf.http_client import AioHttpClient, test_connection
@@ -21,92 +21,68 @@ from evalscope.perf.utils.local_server import start_app
 from evalscope.utils.logger import get_logger
 
 logger = get_logger()
-query_send_completed_event = asyncio.Event()
+
 data_process_completed_event = asyncio.Event()
 
 
 @exception_handler
-async def dispatch_requests_worker(request_queue: asyncio.Queue, args: Arguments):
+async def get_requests(args: Arguments) -> AsyncGenerator[dict, None]:
     query_generator_class = ApiRegistry(args.api)
     query_generator = query_generator_class(args.tokenizer_path)
 
     def load_prompt(prompt_path_or_text):
-        """Load the prompt from a file or directly from the input text."""
         if prompt_path_or_text.startswith('@'):
             with open(prompt_path_or_text[1:], 'r', encoding='utf-8') as file:
                 return file.read()
         return prompt_path_or_text
 
-    async def dispatch_request(request):
-        """Dispatch a single request with optional rate limiting."""
-        await request_queue.put(request)
-        if args.rate != -1:
-            interval = np.random.exponential(1.0 / args.rate)
-            await asyncio.sleep(interval)
-
-    async def dispatch_requests_from_prompt(messages):
-        """Generate and dispatch requests based on the given prompt."""
+    async def generate_requests_from_prompt(messages):
         request = query_generator.build_request(messages, args)
-        if args.number is None:
-            await dispatch_request(request)
-            return 1
         for _ in range(args.number):
-            await dispatch_request(request)
-        return args.number
+            yield request
 
-    async def dispatch_requests_from_dataset():
-        """Generate and dispatch requests based on the dataset."""
-        total_query_count = 0
+    async def generate_requests_from_dataset():
         message_generator_class = DatasetRegistry(args.dataset)
         message_generator = message_generator_class(args)
 
+        count = 0
         for messages in message_generator:
             request = query_generator.build_request(messages, args)
-            if request is None:
-                continue
-            await dispatch_request(request)
-            total_query_count += 1
-            if args.number and total_query_count >= args.number:
-                break
+            if request is not None:
+                yield request
+                count += 1
+                if args.number and count >= args.number:
+                    break
 
-        return total_query_count
-
-    # Load prompt or dataset and dispatch requests accordingly
     if args.prompt:
         prompt = load_prompt(args.prompt)
         messages = [{'role': 'user', 'content': prompt}]
-        total_queries = await dispatch_requests_from_prompt(messages)
+        generator = generate_requests_from_prompt(messages)
     elif args.dataset:
-        total_queries = await dispatch_requests_from_dataset()
+        generator = generate_requests_from_dataset()
     else:
         raise Exception('Either prompt or dataset is required!')
 
-    return total_queries
+    async for request in generator:
+        yield request
+        if args.rate != -1:
+            interval = np.random.exponential(1.0 / args.rate)
+            await asyncio.sleep(interval)
 
 
 @exception_handler
-async def send_requests_worker(
-    task_id,
-    request_queue: asyncio.Queue,
+async def send_request(
+    semaphore: asyncio.Semaphore,
+    request: dict,
     benchmark_data_queue: asyncio.Queue,
     args: Arguments,
 ):
-    client = AioHttpClient(args)
-    async with client:
-        while not (query_send_completed_event.is_set() and request_queue.empty()):
-            try:
-                # Attempt to get a request from the queue with a timeout
-                request = await asyncio.wait_for(request_queue.get(), timeout=0.0001)
-                request_queue.task_done()
-            except asyncio.TimeoutError:
-                # If timeout, continue to the next iteration
-                continue
-
-            # Initialize benchmark data for the current request
+    async with semaphore:
+        client = AioHttpClient(args)
+        async with client:
             benchmark_data = BenchmarkData(request=request)
             collected_messages = []
             try:
-                # Send the request and process the response
                 async for is_error, state_code, response_data in client.post(request):
                     if is_error or state_code != HTTPStatus.OK:
                         logger.error(f'Request: {request} failed, state_code: {state_code}, data: {response_data}')
@@ -124,7 +100,6 @@ async def send_requests_worker(
                 logger.exception(e)
                 logger.error(f'Request query: {request} exception')
             finally:
-                # Record completion time and collected messages
                 benchmark_data.completed_time = time.perf_counter()
                 benchmark_data.response_messages = collected_messages
                 await benchmark_data_queue.put(benchmark_data)
@@ -150,39 +125,45 @@ async def statistic_benchmark_metric_worker(benchmark_data_queue: asyncio.Queue,
         name = args.name if args.name else f'{args.model_id}_{current_time}'
         wandb.init(project='perf_benchmark', name=name, config=args.to_dict())
 
-    with sqlite3.connect(result_db_path) as con:
-        cursor = con.cursor()
-        create_result_table(cursor)
-        with tqdm(desc='Processing') as pbar:
-            while not (data_process_completed_event.is_set() and benchmark_data_queue.empty()):
-                try:
-                    # Attempt to get benchmark data from the queue with a timeout
-                    benchmark_data = await asyncio.wait_for(benchmark_data_queue.get(), timeout=0.01)
-                    benchmark_data_queue.task_done()
-                except asyncio.TimeoutError:
-                    # If timeout, continue to the next iteration
-                    continue
+    collected_benchmark_data = []
+
+    with tqdm(desc='Processing', total=args.number) as pbar:
+        while not (data_process_completed_event.is_set() and benchmark_data_queue.empty()):
+            try:
+                # Attempt to get benchmark data from the queue with a timeout
+                benchmark_data = await asyncio.wait_for(benchmark_data_queue.get(), timeout=0.01)
+                benchmark_data_queue.task_done()
+            except asyncio.TimeoutError:
+                # If timeout, continue to the next iteration
+                continue
 
-                # Update metrics based on the benchmark data
-                metrics.update_metrics(benchmark_data, api_plugin)
+            # Update metrics based on the benchmark data
+            metrics.update_metrics(benchmark_data, api_plugin)
 
-                # Insert benchmark data into the database and commit the transaction
-                insert_benchmark_data(cursor, benchmark_data)
-                con.commit()
+            # Collect benchmark data for later database insertion
+            collected_benchmark_data.append(benchmark_data)
 
-                # Create a message with the updated metrics
-                message = metrics.create_message()
+            # Create a message with the updated metrics
+            message = metrics.create_message()
 
-                # Log the message to wandb if the api key is provided
-                if args.wandb_api_key:
-                    wandb.log(message)
+            # Log the message to wandb if the api key is provided
+            if args.wandb_api_key:
+                wandb.log(message)
 
-                # Log the message to the logger every n queries
-                if int(metrics.n_total_queries) % args.log_every_n_query == 0:
-                    msg = json.dumps(message, ensure_ascii=False, indent=2)
-                    logger.info(msg)
+            # Log the message to the logger every n queries
+            if int(metrics.n_total_queries) % args.log_every_n_query == 0:
+                msg = json.dumps(message, ensure_ascii=False, indent=2)
+                logger.info(msg)
 
-                pbar.update(1)  # Update the progress bar
+            pbar.update(1)  # Update the progress bar
+
+    # Now perform database operations after all benchmark data has been processed
+    with sqlite3.connect(result_db_path) as con:
+        cursor = con.cursor()
+        create_result_table(cursor)
+        for benchmark_data in collected_benchmark_data:
+            insert_benchmark_data(cursor, benchmark_data)
+        con.commit()
 
     return metrics, result_db_path
 
@@ -199,7 +180,7 @@ async def start_server(args: Arguments) -> bool:
         else:
             args.url = f'http://127.0.0.1:{args.port}/v1/chat/completions'
 
-    if not await test_connection(args):
+    if (not args.no_test_connection) and (not await test_connection(args)):
         raise TimeoutError('Test connection failed')
 
 
@@ -210,39 +191,32 @@ async def benchmark(args: Arguments) -> None:
         add_signal_handlers(loop)
 
     # init queue
-    request_queue = asyncio.Queue()
     benchmark_data_queue = asyncio.Queue()
 
     # reset event
-    query_send_completed_event.clear()
     data_process_completed_event.clear()
 
+    semaphore = asyncio.Semaphore(args.parallel)
+
     async def create_send_request_tasks():
         tasks: List[asyncio.Task] = []
-        for idx in range(args.parallel):
-            task = asyncio.create_task(send_requests_worker(idx, request_queue, benchmark_data_queue, args))
+        async for request in get_requests(args):
+            task = asyncio.create_task(send_request(semaphore, request, benchmark_data_queue, args))
             tasks.append(task)
         return tasks
 
     async def run_tasks():
         await start_server(args)
 
-        dispatch_task = asyncio.create_task(dispatch_requests_worker(request_queue, args))
         statistic_benchmark_metric_task = asyncio.create_task(
             statistic_benchmark_metric_worker(benchmark_data_queue, args))
         send_request_tasks = await create_send_request_tasks()
 
-        expected_number_of_queries = await dispatch_task
-        await request_queue.join()
-        query_send_completed_event.set()
-
         await asyncio.gather(*send_request_tasks, return_exceptions=True)
         await benchmark_data_queue.join()
         data_process_completed_event.set()
 
         metrics, result_db_path = await statistic_benchmark_metric_task
-        summary_result(args, metrics, expected_number_of_queries, result_db_path)
-
-        await asyncio.sleep(0.250)
+        summary_result(args, metrics, result_db_path)
 
     await run_tasks()

evalscope/perf/main.py

@@ -32,7 +32,7 @@ def run_perf_benchmark(args):
     if platform.system() == 'Windows':
         asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
 
-    loop = asyncio.get_event_loop()
+    loop = asyncio.new_event_loop()
     if platform.system() != 'Windows':
         add_signal_handlers(loop)
 

evalscope/perf/plugin/api/openai_api.py

@@ -70,7 +70,7 @@ class OpenaiPlugin(ApiPluginBase):
     def __compose_query_from_parameter(self, payload: Dict, param: Arguments):
         payload['model'] = param.model
         if param.max_tokens is not None:
-            payload['max_tokens'] = param.max_tokens
+            payload['max_completion_tokens'] = param.max_tokens
         if param.min_tokens is not None:
             payload['min_tokens'] = param.min_tokens
         if param.frequency_penalty is not None:
@@ -94,9 +94,11 @@ class OpenaiPlugin(ApiPluginBase):
             payload['top_p'] = param.top_p
         if param.top_k is not None:
             payload['top_k'] = param.top_k
+        if param.extra_args is not None:
+            payload.update(param.extra_args)
         return payload
 
-    def parse_responses(self, responses, request: Any = None, **kwargs) -> Dict:
+    def parse_responses(self, responses, request: Any = None, **kwargs) -> tuple[int, int]:
         """Parser responses and return number of request and response tokens.
         Only one response for non-stream, multiple responses for stream.
         """

evalscope/perf/plugin/datasets/__init__.py

@@ -3,4 +3,5 @@ from evalscope.perf.plugin.datasets.flickr8k import FlickrDatasetPlugin
 from evalscope.perf.plugin.datasets.line_by_line import LineByLineDatasetPlugin
 from evalscope.perf.plugin.datasets.longalpaca import LongAlpacaDatasetPlugin
 from evalscope.perf.plugin.datasets.openqa import OpenqaDatasetPlugin
+from evalscope.perf.plugin.datasets.random_dataset import RandomDatasetPlugin
 from evalscope.perf.plugin.datasets.speed_benchmark import SpeedBenchmarkDatasetPlugin, SpeedBenchmarkLongDatasetPlugin

evalscope/perf/plugin/datasets/openqa.py

@@ -1,5 +1,5 @@
 import json
-import subprocess
+import os
 from typing import Any, Dict, Iterator, List
 
 from evalscope.perf.arguments import Arguments
@@ -18,16 +18,11 @@ class OpenqaDatasetPlugin(DatasetPluginBase):
 
     def build_messages(self) -> Iterator[List[Dict]]:
         if not self.query_parameters.dataset_path:
-            subprocess.call([
-                'modelscope',
-                'download',
-                '--dataset',
-                'AI-ModelScope/HC3-Chinese',
-                'open_qa.jsonl',
-                '--local_dir',
-                './data',
-            ])
-            self.query_parameters.dataset_path = './data/open_qa.jsonl'
+            from modelscope import dataset_snapshot_download
+
+            file_name = 'open_qa.jsonl'
+            local_path = dataset_snapshot_download('AI-ModelScope/HC3-Chinese', allow_patterns=[file_name])
+            self.query_parameters.dataset_path = os.path.join(local_path, file_name)
 
         for item in self.dataset_line_by_line(self.query_parameters.dataset_path):
             item = json.loads(item)

evalscope/perf/plugin/datasets/random_dataset.py

@@ -0,0 +1,51 @@
+import numpy as np
+from typing import Dict, Iterator, List
+
+from evalscope.perf.arguments import Arguments
+from evalscope.perf.plugin.datasets.base import DatasetPluginBase
+from evalscope.perf.plugin.registry import register_dataset
+
+
+@register_dataset('random')
+class RandomDatasetPlugin(DatasetPluginBase):
+    """Read dataset and return prompt.
+    """
+
+    def __init__(self, query_parameters: Arguments):
+        super().__init__(query_parameters)
+        assert self.query_parameters.tokenizer_path, 'Tokenizer path is required for random data generation, please provide it with `--tokenizer_path`.'  # noqa: E501
+
+        from modelscope import AutoTokenizer
+        self.tokenizer = AutoTokenizer.from_pretrained(self.query_parameters.tokenizer_path, trust_remote_code=True)
+        self.prefix_length = self.query_parameters.prefix_length
+        self.prefix_ids = self.get_random_inputs(self.prefix_length)
+        self.template_len = self.get_template_len()
+        self.number = self.query_parameters.number or 1
+
+    def build_messages(self) -> Iterator[List[Dict]]:
+        min_prompt_length = self.query_parameters.min_prompt_length - self.template_len
+        max_prompt_length = self.query_parameters.max_prompt_length - self.template_len + 1
+
+        assert min_prompt_length >= 0, f'min_prompt_length should be greater than or equal to the template length {self.template_len}.'  # noqa: E501
+        assert max_prompt_length >= min_prompt_length, 'max_prompt_length should be greater than or equal to min_prompt_length.'  # noqa: E501
+
+        # refer to https://github.com/vllm-project/vllm/blob/ed6e9075d31e32c8548b480a47d1ffb77da1f54c/benchmarks/benchmark_serving.py#L366C1-L399C1  # noqa: E501
+        input_lens = np.random.randint(min_prompt_length, max_prompt_length, size=self.number)
+        offsets = np.random.randint(0, self.tokenizer.vocab_size, size=self.number)
+
+        for i in range(self.number):
+            prompt_ids = (offsets[i] + i + np.arange(input_lens[i])) % self.tokenizer.vocab_size
+            prompt = self.tokenizer.decode(
+                self.prefix_ids + prompt_ids.tolist(), skip_special_tokens=False, clean_up_tokenization_spaces=False)
+            yield [{'role': 'user', 'content': prompt}]
+
+    def get_random_inputs(self, length: int) -> List[int]:
+        if length <= 0:
+            return []
+        input_ids = np.random.randint(0, self.tokenizer.vocab_size, size=length).tolist()
+        return input_ids
+
+    def get_template_len(self):
+        empty_message = [{'role': 'user', 'content': ''}]
+        template = self.tokenizer.apply_chat_template(empty_message, tokenize=True, add_generation_prompt=True)
+        return len(template)

evalscope/perf/plugin/datasets/speed_benchmark.py

@@ -3,6 +3,9 @@ from typing import Dict, Iterator, List, Tuple
 from evalscope.perf.arguments import Arguments
 from evalscope.perf.plugin.datasets.base import DatasetPluginBase
 from evalscope.perf.plugin.registry import register_dataset
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
 
 
 @register_dataset('speed_benchmark')
@@ -18,6 +21,14 @@ class SpeedBenchmarkDatasetPlugin(DatasetPluginBase):
     def __init__(self, query_parameters: Arguments):
         super().__init__(query_parameters)
 
+        url = self.query_parameters.url
+        if url.endswith('v1/chat/completions'):
+            logger.warning(
+                'The API URL is not set correctly for `speed_benchmark`. Using `v1/completions` instead of `v1/chat/completions` by system.'  # noqa
+            )
+            url = url.replace('v1/chat/completions', 'v1/completions')
+            self.query_parameters.url = url
+
     def build_messages(self) -> Iterator[List[Dict]]:
         for input_len in self.INPUT_LENGTH:
             for _ in range(self.REPEAT):

evalscope/perf/utils/db_util.py

@@ -2,6 +2,7 @@ import base64
 import json
 import os
 import pickle
+import re
 import sqlite3
 import sys
 from datetime import datetime
@@ -91,6 +92,8 @@ def insert_benchmark_data(cursor: sqlite3.Cursor, benchmark_data: BenchmarkData)
 def get_output_path(args: Arguments) -> str:
     current_time = datetime.now().strftime('%Y%m%d_%H%M%S')
     output_path = os.path.join(args.outputs_dir, current_time, f'{args.name or args.model_id}')
+    # Filter illegal characters
+    output_path = re.sub(r'[<>:"|?*]', '_', output_path)
     if not os.path.exists(output_path):
         os.makedirs(output_path, exist_ok=True)
     logger.info(f'Save the result to: {output_path}')
@@ -191,12 +194,12 @@ def get_percentile_results(result_db_path: str) -> Dict[str, List[float]]:
     return results
 
 
-def summary_result(args: Arguments, metrics: BenchmarkMetrics, expected_number_of_queries: int, result_db_path: str):
+def summary_result(args: Arguments, metrics: BenchmarkMetrics, result_db_path: str):
     result_path = os.path.dirname(result_db_path)
     write_json_file(args.to_dict(), os.path.join(result_path, 'benchmark_args.json'))
 
     data = metrics.create_message()
-    data.update({'Expected number of requests': expected_number_of_queries, 'Result DB path': result_db_path})
+    data.update({'Expected number of requests': args.number, 'Result DB path': result_db_path})
     write_json_file(data, os.path.join(result_path, 'benchmark_summary.json'))
 
     # Print summary in a table

evalscope/run.py

@@ -39,9 +39,11 @@ def run_single_task(task_cfg: TaskConfig, run_time: str) -> dict:
     configure_logging(task_cfg.debug, os.path.join(outputs.logs_dir, 'eval_log.log'))
 
     if task_cfg.eval_backend != EvalBackend.NATIVE:
-        return run_non_native_backend(task_cfg, outputs)
+        result = run_non_native_backend(task_cfg, outputs)
     else:
-        return evaluate_model(task_cfg, outputs)
+        result = evaluate_model(task_cfg, outputs)
+
+    return result
 
 
 def setup_work_directory(task_cfg: TaskConfig, run_time: str):
@@ -117,6 +119,16 @@ def evaluate_model(task_cfg: TaskConfig, outputs: OutputsStructure) -> dict:
         res_dict = evaluator.eval()
         eval_results[evaluator.dataset_name] = res_dict
 
+    # Clean up
+    if base_model is not None:
+        import gc
+        import torch
+
+        del base_model
+        del evaluators
+        torch.cuda.empty_cache()
+        gc.collect()
+
     return eval_results
 
 

evalscope/version.py

@@ -1,4 +1,4 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 
-__version__ = '0.13.0'
-__release_datetime__ = '2025-03-14 12:00:00'
+__version__ = '0.13.2'
+__release_datetime__ = '2025-04-01 20:00:00'