evalscope 0.10.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

evalscope/perf/__init__.py

@@ -1 +0,0 @@
-from evalscope.perf.main import run_perf_benchmark

evalscope/perf/arguments.py

@@ -3,13 +3,14 @@ import json
 import os
 import sys
 from dataclasses import dataclass, field
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List, Optional, Union
 
 from evalscope.constants import DEFAULT_WORK_DIR
+from evalscope.utils import BaseArgument
 
 
 @dataclass
-class Arguments:
+class Arguments(BaseArgument):
     # Model and API
     model: str  # Model name or path
     model_id: Optional[str] = None  # Model identifier
@@ -21,29 +22,46 @@ class Arguments:
     # Connection settings
     url: str = 'http://127.0.0.1:8877/v1/chat/completions'  # URL for the API connection
     headers: Dict[str, Any] = field(default_factory=dict)  # Custom headers
-    connect_timeout: int = 120  # Connection timeout in seconds
-    read_timeout: int = 120  # Read timeout in seconds
-    api_key: str = 'EMPTY'
+    connect_timeout: int = 600  # Connection timeout in seconds
+    read_timeout: int = 600  # Read timeout in seconds
+    api_key: Optional[str] = None
+    no_test_connection: bool = False  # Test the connection before starting the benchmark
 
     # Performance and parallelism
-    number: Optional[int] = None  # Number of requests to be made
-    parallel: int = 1  # Number of parallel requests
+    number: Union[int, List[int]] = 1000  # Number of requests to be made
+    parallel: Union[int, List[int]] = 1  # Number of parallel requests
     rate: int = -1  # Rate limit for requests (default: -1, no limit)
+    sleep_interval: int = 5  # Sleep interval between performance runs, in seconds
+
+    # Tuning knobs
+    db_commit_interval: int = 1000  # Number of rows buffered before committing to the DB
+    queue_size_multiplier: int = 5  # Maxsize for queue = parallel * this multiplier
+    in_flight_task_multiplier: int = 2  # Max scheduled tasks = parallel * this multiplier
 
     # Logging and debugging
     log_every_n_query: int = 10  # Log every N queries
     debug: bool = False  # Debug mode
-    wandb_api_key: Optional[str] = None  # WandB API key for logging
+    visualizer: Optional[str] = None  # Visualizer for logging, supports 'swanlab' or 'wandb'
+    wandb_api_key: Optional[str] = None  # Will be deprecated in the future
+    swanlab_api_key: Optional[str] = None  # Will be deprecated in the future
     name: Optional[str] = None  # Name for the run
 
     # Output settings
     outputs_dir: str = DEFAULT_WORK_DIR
 
     # Prompt settings
-    max_prompt_length: int = sys.maxsize  # Maximum length of the prompt
+    max_prompt_length: int = 131072  # Maximum length of the prompt
     min_prompt_length: int = 0  # Minimum length of the prompt
+    prefix_length: int = 0  # Length of the prefix, only for random dataset
     prompt: Optional[str] = None  # The prompt text
     query_template: Optional[str] = None  # Template for the query
+    apply_chat_template: Optional[bool] = None  # Whether to apply chat template
+    # random vl settings
+    image_width: int = 224  # Width of the image for random VL dataset
+    image_height: int = 224  # Height of the image for random VL dataset
+    image_format: str = 'RGB'  # Image format for random VL dataset
+    image_num: int = 1  # Number of images for random VL dataset
+    image_patch_size: int = 28  # Patch size for image tokenizer, only for local image token calculation
 
     # Dataset settings
     dataset: str = 'openqa'  # Dataset type (default: 'line_by_line')
@@ -51,68 +69,57 @@ class Arguments:
 
     # Response settings
     frequency_penalty: Optional[float] = None  # Frequency penalty for the response
+    repetition_penalty: Optional[float] = None  # Repetition penalty for the response
     logprobs: Optional[bool] = None  # Whether to log probabilities
     max_tokens: Optional[int] = 2048  # Maximum number of tokens in the response
     min_tokens: Optional[int] = None  # Minimum number of tokens in the response
     n_choices: Optional[int] = None  # Number of response choices
-    seed: Optional[int] = 42  # Random seed for reproducibility
-    stop: Optional[List[str]] = field(default_factory=list)  # Stop sequences for the response
-    stop_token_ids: Optional[List[str]] = field(default_factory=list)  # Stop token IDs for the response
-    stream: Optional[bool] = None  # Whether to stream the response
-    temperature: Optional[float] = None  # Temperature setting for the response
+    seed: Optional[int] = None  # Random seed for reproducibility
+    stop: Optional[List[str]] = None  # Stop sequences for the response
+    stop_token_ids: Optional[List[str]] = None  # Stop token IDs for the response
+    stream: Optional[bool] = True  # Whether to stream the response
+    temperature: float = 0.0  # Temperature setting for the response
     top_p: Optional[float] = None  # Top-p (nucleus) sampling setting for the response
-
-    @staticmethod
-    def from_args(args):
-        return Arguments(
-            model=args.model,
-            attn_implementation=args.attn_implementation,
-            url=args.url,
-            port=args.port,
-            api_key=args.api_key,
-            connect_timeout=args.connect_timeout,
-            read_timeout=args.read_timeout,
-            number=args.number,
-            parallel=args.parallel,
-            rate=args.rate,
-            log_every_n_query=args.log_every_n_query,
-            headers=args.headers,
-            wandb_api_key=args.wandb_api_key,
-            name=args.name,
-            outputs_dir=args.outputs_dir,
-            debug=args.debug,
-            tokenizer_path=args.tokenizer_path,
-            api=args.api,
-            max_prompt_length=args.max_prompt_length,
-            min_prompt_length=args.min_prompt_length,
-            prompt=args.prompt,
-            query_template=args.query_template,
-            dataset=args.dataset,
-            dataset_path=args.dataset_path,
-            frequency_penalty=args.frequency_penalty,
-            logprobs=args.logprobs,
-            max_tokens=args.max_tokens,
-            min_tokens=args.min_tokens,
-            n_choices=args.n_choices,
-            seed=args.seed,
-            stop=args.stop,
-            stop_token_ids=args.stop_token_ids,
-            stream=args.stream,
-            temperature=args.temperature,
-            top_p=args.top_p)
+    top_k: Optional[int] = None  # Top-k sampling setting for the response
+    extra_args: Optional[Dict[str, Any]] = None  # Extra arguments
 
     def __post_init__(self):
+        # Set the default headers
         self.headers = self.headers or {}  # Default to empty dictionary
         if self.api_key:
             # Assuming the API key is used as a Bearer token
             self.headers['Authorization'] = f'Bearer {self.api_key}'
-        self.model_id = os.path.basename(self.model)
 
-    def __str__(self):
-        return json.dumps(self.to_dict(), indent=4, default=str, ensure_ascii=False)
+        # Set the model ID based on the model name
+        self.model_id = os.path.basename(self.model)
 
-    def to_dict(self) -> Dict[str, Any]:
-        return self.__dict__
+        # Set the URL based on the dataset type
+        if self.api.startswith('local'):
+            if self.dataset.startswith('speed_benchmark'):
+                self.url = f'http://127.0.0.1:{self.port}/v1/completions'
+            else:
+                self.url = f'http://127.0.0.1:{self.port}/v1/chat/completions'
+
+        # Set the apply_chat_template flag based on the URL
+        if self.apply_chat_template is None:
+            self.apply_chat_template = self.url.strip('/').endswith('chat/completions')
+
+        # Set number and parallel to lists if they are integers
+        if isinstance(self.number, int):
+            self.number = [self.number]
+        if isinstance(self.parallel, int):
+            self.parallel = [self.parallel]
+        assert len(self.number) == len(
+            self.parallel
+        ), f'The length of number and parallel should be the same, but got number: {self.number} and parallel: {self.parallel}'  # noqa: E501
+
+        # Validate tuning knobs
+        if self.db_commit_interval <= 0:
+            self.db_commit_interval = 1
+        if self.queue_size_multiplier <= 0:
+            self.queue_size_multiplier = 1
+        if self.in_flight_task_multiplier <= 0:
+            self.in_flight_task_multiplier = 1
 
 
 class ParseKVAction(argparse.Action):
@@ -122,7 +129,13 @@ class ParseKVAction(argparse.Action):
             setattr(namespace, self.dest, {})
         else:
             try:
-                kv_dict = dict(kv.split('=') for kv in values)
+                kv_dict = {}
+                for kv in values:
+                    parts = kv.split('=', 1)  # only split the first '='
+                    if len(parts) != 2:
+                        raise ValueError(f'Invalid key-value pair: {kv}')
+                    key, value = parts
+                    kv_dict[key.strip()] = value.strip()
                 setattr(namespace, self.dest, kv_dict)
             except ValueError as e:
                 parser.error(f'Error parsing key-value pairs: {e}')
@@ -141,26 +154,45 @@ def add_argument(parser: argparse.ArgumentParser):
     parser.add_argument('--url', type=str, default='http://127.0.0.1:8877/v1/chat/completions')
     parser.add_argument('--port', type=int, default=8877, help='The port for local inference')
     parser.add_argument('--headers', nargs='+', dest='headers', action=ParseKVAction, help='Extra HTTP headers')
-    parser.add_argument('--api-key', type=str, required=False, default='EMPTY', help='The API key for authentication')
-    parser.add_argument('--connect-timeout', type=int, default=120, help='The network connection timeout')
-    parser.add_argument('--read-timeout', type=int, default=120, help='The network read timeout')
+    parser.add_argument('--api-key', type=str, required=False, default=None, help='The API key for authentication')
+    parser.add_argument('--connect-timeout', type=int, default=600, help='The network connection timeout')
+    parser.add_argument('--read-timeout', type=int, default=600, help='The network read timeout')
+    parser.add_argument('--no-test-connection', action='store_false', default=False, help='Do not test the connection before starting the benchmark')  # noqa: E501
 
     # Performance and parallelism
-    parser.add_argument('-n', '--number', type=int, default=None, help='How many requests to be made')
-    parser.add_argument('--parallel', type=int, default=1, help='Set number of concurrency requests, default 1')
+    parser.add_argument('-n', '--number', type=int, default=1000, nargs='+', help='How many requests to be made')
+    parser.add_argument('--parallel', type=int, default=1, nargs='+', help='Set number of concurrency requests, default 1')  # noqa: E501
     parser.add_argument('--rate', type=int, default=-1, help='Number of requests per second. default None')
+    parser.add_argument(
+        '--sleep-interval', type=int, default=5, help='Sleep interval between performance runs, in seconds. Default 5')  # noqa: E501
+
+    # Tuning knobs
+    parser.add_argument('--db-commit-interval', type=int, default=1000, help='Rows buffered before SQLite commit')
+    parser.add_argument('--queue-size-multiplier', type=int, default=5, help='Queue maxsize = parallel * multiplier')
+    parser.add_argument('--in-flight-task-multiplier', type=int, default=2, help='Max scheduled tasks = parallel * multiplier')  # noqa: E501
 
     # Logging and debugging
     parser.add_argument('--log-every-n-query', type=int, default=10, help='Logging every n query')
     parser.add_argument('--debug', action='store_true', default=False, help='Debug request send')
+    parser.add_argument('--visualizer', type=str, default=None, help='The visualizer to use, default None')
     parser.add_argument('--wandb-api-key', type=str, default=None, help='The wandb API key')
-    parser.add_argument('--name', type=str, help='The wandb db result name and result db name')
+    parser.add_argument('--swanlab-api-key', type=str, default=None, help='The swanlab API key')
+    parser.add_argument('--name', type=str, help='The wandb/swanlab db result name and result db name')
 
     # Prompt settings
     parser.add_argument('--max-prompt-length', type=int, default=sys.maxsize, help='Maximum input prompt length')
     parser.add_argument('--min-prompt-length', type=int, default=0, help='Minimum input prompt length')
+    parser.add_argument('--prefix-length', type=int, default=0, help='The prefix length')
     parser.add_argument('--prompt', type=str, required=False, default=None, help='Specified the request prompt')
     parser.add_argument('--query-template', type=str, default=None, help='Specify the query template')
+    parser.add_argument(
+        '--apply-chat-template', type=argparse.BooleanOptionalAction, default=None, help='Apply chat template to the prompt')  # noqa: E501
+    # random vl settings
+    parser.add_argument('--image-width', type=int, default=224, help='Width of the image for random VL dataset')
+    parser.add_argument('--image-height', type=int, default=224, help='Height of the image for random VL dataset')
+    parser.add_argument('--image-format', type=str, default='RGB', help='Image format for random VL dataset')
+    parser.add_argument('--image-num', type=int, default=1, help='Number of images for random VL dataset')
+    parser.add_argument('--image-patch-size', type=int, default=28, help='Patch size for image tokenizer, only for local image token calculation')  # noqa: E501
 
     # Output settings
     parser.add_argument('--outputs-dir', help='Outputs dir.', default='outputs')
@@ -171,19 +203,21 @@ def add_argument(parser: argparse.ArgumentParser):
 
     # Response settings
     parser.add_argument('--frequency-penalty', type=float, help='The frequency_penalty value', default=None)
+    parser.add_argument('--repetition-penalty', type=float, help='The repetition_penalty value', default=None)
     parser.add_argument('--logprobs', action='store_true', help='The logprobs', default=None)
     parser.add_argument(
         '--max-tokens', type=int, help='The maximum number of tokens that can be generated', default=2048)
     parser.add_argument(
         '--min-tokens', type=int, help='The minimum number of tokens that can be generated', default=None)
     parser.add_argument('--n-choices', type=int, help='How many completion choices to generate', default=None)
-    parser.add_argument('--seed', type=int, help='The random seed', default=42)
+    parser.add_argument('--seed', type=int, help='The random seed', default=None)
     parser.add_argument('--stop', nargs='*', help='The stop tokens', default=None)
     parser.add_argument('--stop-token-ids', nargs='*', help='Set the stop token IDs', default=None)
-    parser.add_argument('--stream', action='store_true', help='Stream output with SSE', default=None)
-    parser.add_argument('--temperature', type=float, help='The sample temperature', default=None)
+    parser.add_argument('--stream', action=argparse.BooleanOptionalAction, help='Stream output with SSE', default=True)
+    parser.add_argument('--temperature', type=float, help='The sample temperature', default=0.0)
     parser.add_argument('--top-p', type=float, help='Sampling top p', default=None)
-
+    parser.add_argument('--top-k', type=int, help='Sampling top k', default=None)
+    parser.add_argument('--extra-args', type=json.loads, default='{}', help='Extra arguments, should in JSON format',)
     # yapf: enable
 
 

evalscope/perf/benchmark.py

@@ -1,243 +1,194 @@
 import asyncio
-import copy
 import json
 import numpy as np
-import os
 import platform
 import sqlite3
-import threading
-import time
-from http import HTTPStatus
 from tqdm import tqdm
-from typing import List
-
-from evalscope.perf.arguments import Arguments
-from evalscope.perf.http_client import AioHttpClient, test_connection
-from evalscope.perf.plugin.registry import ApiRegistry, DatasetRegistry
-from evalscope.perf.utils.benchmark_util import BenchmarkData, BenchmarkMetrics
-from evalscope.perf.utils.db_util import create_result_table, get_result_db_path, insert_benchmark_data, summary_result
-from evalscope.perf.utils.handler import add_signal_handlers, exception_handler
-from evalscope.perf.utils.local_server import start_app
+from typing import TYPE_CHECKING, AsyncGenerator, Dict, List, Tuple
+
 from evalscope.utils.logger import get_logger
+from .arguments import Arguments
+from .http_client import AioHttpClient, test_connection
+from .plugin import ApiRegistry, DatasetRegistry
+from .utils.benchmark_util import BenchmarkData, BenchmarkMetrics
+from .utils.db_util import create_result_table, get_result_db_path, insert_benchmark_data, load_prompt, summary_result
+from .utils.handler import add_signal_handlers, exception_handler
+
+if TYPE_CHECKING:
+    from .plugin import ApiPluginBase, DatasetPluginBase
 
 logger = get_logger()
-query_send_completed_event = asyncio.Event()
+
 data_process_completed_event = asyncio.Event()
 
 
 @exception_handler
-async def dispatch_requests_worker(request_queue: asyncio.Queue, args: Arguments):
-    query_generator_class = ApiRegistry(args.api)
-    query_generator = query_generator_class(args.tokenizer_path)
-
-    def load_prompt(prompt_path_or_text):
-        """Load the prompt from a file or directly from the input text."""
-        if prompt_path_or_text.startswith('@'):
-            with open(prompt_path_or_text[1:], 'r', encoding='utf-8') as file:
-                return file.read()
-        return prompt_path_or_text
-
-    async def dispatch_request(request):
-        """Dispatch a single request with optional rate limiting."""
-        await request_queue.put(request)
-        if args.rate != -1:
-            interval = np.random.exponential(1.0 / args.rate)
-            await asyncio.sleep(interval)
+async def get_requests(args: Arguments, api_plugin: 'ApiPluginBase') -> AsyncGenerator[dict, None]:
 
-    async def dispatch_requests_from_prompt(messages):
-        """Generate and dispatch requests based on the given prompt."""
-        request = query_generator.build_request(messages, args)
-        if args.number is None:
-            await dispatch_request(request)
-            return 1
+    async def generate_requests_from_prompt():
+        prompt = load_prompt(args.prompt)
+        messages = [{'role': 'user', 'content': prompt}] if args.apply_chat_template else prompt
+        request = api_plugin.build_request(messages)
         for _ in range(args.number):
-            await dispatch_request(request)
-        return args.number
+            yield request
 
-    async def dispatch_requests_from_dataset():
-        """Generate and dispatch requests based on the dataset."""
-        total_query_count = 0
-        message_generator_class = DatasetRegistry(args.dataset)
+    async def generate_requests_from_dataset():
+        message_generator_class = DatasetRegistry.get_class(args.dataset)
         message_generator = message_generator_class(args)
 
-        for messages in message_generator:
-            request = query_generator.build_request(messages, args)
-            if request is None:
-                continue
-            await dispatch_request(request)
-            total_query_count += 1
-            if args.number and total_query_count >= args.number:
-                break
+        dataset_messages = []
+        try:
+            for messages in message_generator.build_messages():
+                dataset_messages.append(messages)
+                if len(dataset_messages) >= args.number:
+                    break
+        except StopIteration:
+            pass
 
-        return total_query_count
+        if not dataset_messages:
+            raise Exception('Dataset is empty!')
+
+        count = 0
+        dataset_index = 0
+
+        while count < args.number:
+            messages = dataset_messages[dataset_index]
+            request = api_plugin.build_request(messages)
+            if request is not None:
+                yield request
+                count += 1
+
+            dataset_index = (dataset_index + 1) % len(dataset_messages)
 
-    # Load prompt or dataset and dispatch requests accordingly
     if args.prompt:
-        prompt = load_prompt(args.prompt)
-        messages = [{'role': 'user', 'content': prompt}]
-        total_queries = await dispatch_requests_from_prompt(messages)
+        generator = generate_requests_from_prompt()
     elif args.dataset:
-        total_queries = await dispatch_requests_from_dataset()
+        generator = generate_requests_from_dataset()
     else:
-        raise Exception('Either prompt or dataset is required!')
+        raise ValueError('Either prompt or dataset is required!')
 
-    return total_queries
+    async for request in generator:
+        yield request
+        if args.rate != -1:
+            interval = np.random.exponential(1.0 / args.rate)
+            await asyncio.sleep(interval)
 
 
 @exception_handler
-async def send_requests_worker(
-    task_id,
-    request_queue: asyncio.Queue,
+async def send_request(
+    semaphore: asyncio.Semaphore,
+    request: dict,
     benchmark_data_queue: asyncio.Queue,
     args: Arguments,
+    client: AioHttpClient,  # reuse shared client
 ):
-    client = AioHttpClient(args)
-    async with client:
-        while not (query_send_completed_event.is_set() and request_queue.empty()):
-            try:
-                # Attempt to get a request from the queue with a timeout
-                request = await asyncio.wait_for(request_queue.get(), timeout=0.0001)
-                request_queue.task_done()
-            except asyncio.TimeoutError:
-                # If timeout, continue to the next iteration
-                continue
-
-            # Initialize benchmark data for the current request
-            benchmark_data = BenchmarkData(request=request)
-            collected_messages = []
-            try:
-                # Send the request and process the response
-                async for is_error, state_code, response_data in client.post(request):
-                    if is_error or state_code != HTTPStatus.OK:
-                        logger.error(f'Request: {request} failed, state_code: {state_code}, data: {response_data}')
-                        benchmark_data.success = False
-                        break
-                    if response_data:
-                        collected_messages.append(response_data)
-                        benchmark_data.chunk_times.append(time.perf_counter())
-                        benchmark_data.success = True
-                        benchmark_data.update_gpu_usage()
-            except Exception as e:
-                if response_data:
-                    collected_messages.append(response_data)
-                benchmark_data.success = False
-                logger.exception(e)
-                logger.error(f'Request query: {request} exception')
-            finally:
-                # Record completion time and collected messages
-                benchmark_data.completed_time = time.perf_counter()
-                benchmark_data.response_messages = collected_messages
-                await benchmark_data_queue.put(benchmark_data)
+    async with semaphore:
+        benchmark_data = await client.post(request)
+        benchmark_data.update_gpu_usage()
+        await benchmark_data_queue.put(benchmark_data)
 
 
 @exception_handler
-async def statistic_benchmark_metric_worker(benchmark_data_queue: asyncio.Queue, args: Arguments):
+async def statistic_benchmark_metric(benchmark_data_queue: asyncio.Queue, args: Arguments, api_plugin: 'ApiPluginBase'):
     metrics = BenchmarkMetrics(concurrency=args.parallel)
-
-    api_plugin_class = ApiRegistry(args.api)
-    api_plugin = api_plugin_class(args.tokenizer_path)
-
     result_db_path = get_result_db_path(args)
-    # Initialize wandb
-    if args.wandb_api_key:
-        import datetime
-        import wandb
-        os.environ['WANDB_SILENT'] = 'true'
-        os.environ['WANDB_DIR'] = args.outputs_dir
-
-        wandb.login(key=args.wandb_api_key)
-        current_time = datetime.datetime.now().strftime('%Y%m%d_%H%M%S')
-        name = args.name if args.name else f'{args.model_id}_{current_time}'
-        wandb.init(project='perf_benchmark', name=name, config=args.to_dict())
+
+    # Stream inserts to DB to avoid accumulating all results in memory
+    commit_every = args.db_commit_interval
+    processed_since_commit = 0
 
     with sqlite3.connect(result_db_path) as con:
         cursor = con.cursor()
         create_result_table(cursor)
-        with tqdm(desc='Processing') as pbar:
+
+        with tqdm(desc='Processing', total=args.number) as pbar:
             while not (data_process_completed_event.is_set() and benchmark_data_queue.empty()):
                 try:
-                    # Attempt to get benchmark data from the queue with a timeout
-                    benchmark_data = await asyncio.wait_for(benchmark_data_queue.get(), timeout=0.01)
-                    benchmark_data_queue.task_done()
+                    benchmark_data = await asyncio.wait_for(benchmark_data_queue.get(), timeout=0.1)
                 except asyncio.TimeoutError:
-                    # If timeout, continue to the next iteration
                     continue
 
-                # Update metrics based on the benchmark data
+                # Update metrics and write to DB immediately
                 metrics.update_metrics(benchmark_data, api_plugin)
-
-                # Insert benchmark data into the database and commit the transaction
                 insert_benchmark_data(cursor, benchmark_data)
-                con.commit()
+                processed_since_commit += 1
+                if processed_since_commit >= commit_every:
+                    con.commit()
+                    processed_since_commit = 0
 
-                # Create a message with the updated metrics
                 message = metrics.create_message()
 
-                # Log the message to wandb if the api key is provided
                 if args.wandb_api_key:
+                    import wandb
                     wandb.log(message)
+                if args.swanlab_api_key:
+                    import swanlab
+                    swanlab.log(message)
 
-                # Log the message to the logger every n queries
                 if int(metrics.n_total_queries) % args.log_every_n_query == 0:
                     msg = json.dumps(message, ensure_ascii=False, indent=2)
                     logger.info(msg)
 
-                pbar.update(1)  # Update the progress bar
+                benchmark_data_queue.task_done()
+                pbar.update(1)
+
+        con.commit()
 
     return metrics, result_db_path
 
 
 @exception_handler
-async def start_server(args: Arguments) -> bool:
-    if args.api.startswith('local'):
-        #  start local server
-        server = threading.Thread(target=start_app, args=(copy.deepcopy(args), ), daemon=True)
-        server.start()
-
-        if args.dataset.startswith('speed_benchmark'):
-            args.url = f'http://127.0.0.1:{args.port}/v1/completions'
-        else:
-            args.url = f'http://127.0.0.1:{args.port}/v1/chat/completions'
-
-    if not await test_connection(args):
+async def connect_test(args: Arguments, api_plugin) -> bool:
+    if (not args.no_test_connection) and (not await test_connection(args, api_plugin)):
         raise TimeoutError('Test connection failed')
 
 
 @exception_handler
-async def benchmark(args: Arguments) -> None:
+async def benchmark(args: Arguments) -> Tuple[Dict, Dict]:
     if platform.system() != 'Windows':
         loop = asyncio.get_running_loop()
         add_signal_handlers(loop)
 
-    request_queue = asyncio.Queue()
-    benchmark_data_queue = asyncio.Queue()
+    api_plugin_class = ApiRegistry.get_class(args.api)
+    api_plugin = api_plugin_class(args)
 
-    async def create_send_request_tasks():
-        tasks: List[asyncio.Task] = []
-        for idx in range(args.parallel):
-            task = asyncio.create_task(send_requests_worker(idx, request_queue, benchmark_data_queue, args))
-            tasks.append(task)
-        return tasks
+    benchmark_data_queue: asyncio.Queue = asyncio.Queue(maxsize=max(1, args.parallel * args.queue_size_multiplier))
+    data_process_completed_event.clear()
 
-    async def run_tasks():
-        await start_server(args)
+    # test connection
+    await connect_test(args, api_plugin)
 
-        dispatch_task = asyncio.create_task(dispatch_requests_worker(request_queue, args))
+    # Create a single shared client session for all requests
+    client = AioHttpClient(args, api_plugin)
+    async with client:
+        # start statistic benchmark metric (consumer)
         statistic_benchmark_metric_task = asyncio.create_task(
-            statistic_benchmark_metric_worker(benchmark_data_queue, args))
-        send_request_tasks = await create_send_request_tasks()
+            statistic_benchmark_metric(benchmark_data_queue, args, api_plugin)
+        )
+
+        # start sending requests with bounded in-flight tasks
+        semaphore = asyncio.Semaphore(args.parallel)
+        in_flight: set[asyncio.Task] = set()
+        max_in_flight = args.parallel * args.in_flight_task_multiplier
 
-        expected_number_of_queries = await dispatch_task
-        await request_queue.join()
-        query_send_completed_event.set()
+        async for request in get_requests(args, api_plugin):
+            # Keep the number of scheduled tasks bounded to avoid OOM
+            if len(in_flight) >= max_in_flight:
+                done, pending = await asyncio.wait(in_flight, return_when=asyncio.FIRST_COMPLETED)
+                in_flight = pending
 
-        await asyncio.gather(*send_request_tasks, return_exceptions=True)
+            task = asyncio.create_task(send_request(semaphore, request, benchmark_data_queue, args, client))
+            in_flight.add(task)
+
+        # Wait for remaining in-flight tasks
+        if in_flight:
+            await asyncio.gather(*in_flight, return_exceptions=True)
+
+        # Drain queue and finish
         await benchmark_data_queue.join()
         data_process_completed_event.set()
 
         metrics, result_db_path = await statistic_benchmark_metric_task
-        summary_result(args, metrics, expected_number_of_queries, result_db_path)
-
-        await asyncio.sleep(0.250)
 
-    await run_tasks()
+    metrics_result, percentile_result = summary_result(args, metrics, result_db_path)
+    return metrics_result, percentile_result