evalscope 0.15.1__py3-none-any.whl → 0.16.1__py3-none-any.whl

evalscope/models/adapters/chat_adapter.py

@@ -1,7 +1,7 @@
 import os
 import time
 import torch
-from typing import Any, Dict, List, Tuple, Union
+from typing import Any, Dict, List, Optional, Tuple, Union
 
 from evalscope.utils.chat_service import ChatCompletionResponse, ChatCompletionResponseChoice, ChatMessage, Usage
 from evalscope.utils.logger import get_logger
@@ -58,19 +58,15 @@ class ChatGenerationModelAdapter(BaseModelAdapter):
         return generation_config
 
     def _model_generate(self,
-                        queries: List[str],
-                        system_prompts: List[str] = None,
+                        formatted_prompts: List[str],
                         infer_cfg: Dict[str, Any] = None) -> Tuple[List[List[str]], List[int]]:
         """
         Args:
-            queries: The input queries.
-            system_prompts: The system prompts.
+            formatted_prompts: The formatted prompts.
             infer_cfg: The inference configuration.
         Returns:
             The prediction results.
         """
-        if system_prompts is None:
-            system_prompts = []
         if infer_cfg is None:
             infer_cfg = {}
 
@@ -92,27 +88,6 @@ class ChatGenerationModelAdapter(BaseModelAdapter):
         self.generation_config.update(**infer_cfg)
         fix_do_sample_warning(self.generation_config)
 
-        # For chat model, use the chat template to format the input
-        if self.tokenizer.chat_template is not None:
-            formatted_prompts = []
-            for i, query in enumerate(queries):
-                messages = [ChatMessage(role='user', content=query)]
-                if i < len(system_prompts) and system_prompts[i]:
-                    messages = [ChatMessage(role='system', content=system_prompts[i])] + messages
-                # whether thinking is needed
-                chat_template_kwargs = infer_cfg.get('chat_template_kwargs', None)
-                if chat_template_kwargs is not None:
-                    prompts = self.tokenizer.apply_chat_template(
-                        messages, tokenize=False, add_generation_prompt=True, **chat_template_kwargs)
-                else:
-                    prompts = self.tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
-                formatted_prompts.append(prompts)
-        else:
-            # For base model, use the queries as the input
-            formatted_prompts = queries
-
-        logger.debug(f'formatted_prompts: {formatted_prompts}')
-
         # Get input ids
         inputs = self.tokenizer(
             formatted_prompts, return_tensors='pt', padding=True, truncation=True,
@@ -136,26 +111,68 @@ class ChatGenerationModelAdapter(BaseModelAdapter):
 
         return responses, input_lengths
 
-    @torch.no_grad()
-    def predict(self, inputs: List[dict], infer_cfg: dict = {}) -> List[dict]:
+    def _prepare_inputs(self, inputs: List[dict], infer_cfg: dict = {}) -> List[str]:
         """
+        Prepare the inputs for the model.
         Args:
             inputs: The input data.
             infer_cfg: The inference configuration.
         Returns:
-            The prediction results.
+            The prepared inputs and system prompts.
         """
-
-        # Process inputs
         queries = []
         system_prompts = []
+        message_list = []
 
         for input_item in inputs:
             queries.append(input_item['data'][0])
             system_prompts.append(input_item.get('system_prompt', None))
+            if input_item.get('messages', None):
+                message_list.append(input_item.get('messages', None))
+
+        # For non chat model, use the original queries as the input
+        if self.tokenizer.chat_template is None:
+            return queries
+
+        # For chat model, use the messages as the input
+        # if message_list is None, use the queries as the input
+        if len(message_list) == 0:
+            for i, query in enumerate(queries):
+                messages = [ChatMessage(role='user', content=query)]
+                if i < len(system_prompts) and system_prompts[i]:
+                    messages = [ChatMessage(role='system', content=system_prompts[i])] + messages
+                message_list.append(messages)
+
+        # Format the messages
+        formatted_prompts = []
+        for messages in message_list:
+            # apply chat template
+            chat_template_kwargs = infer_cfg.get('chat_template_kwargs', None)
+            if chat_template_kwargs is not None:
+                prompts = self.tokenizer.apply_chat_template(
+                    messages, tokenize=False, add_generation_prompt=True, **chat_template_kwargs)
+            else:
+                prompts = self.tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
+            formatted_prompts.append(prompts)
+
+        logger.debug(f'formatted_prompts: {formatted_prompts}')
+        return formatted_prompts
+
+    @torch.no_grad()
+    def predict(self, inputs: List[dict], infer_cfg: Optional[dict] = {}) -> List[dict]:
+        """
+        Args:
+            inputs: The input data.
+            infer_cfg: The inference configuration.
+        Returns:
+            The prediction results.
+        """
+
+        # Process inputs
+        formatted_prompts = self._prepare_inputs(inputs, infer_cfg)
 
         # Run inference
-        responses, input_lengths = self._model_generate(queries, system_prompts, infer_cfg)
+        responses, input_lengths = self._model_generate(formatted_prompts, infer_cfg)
 
         # Process outputs
         results = []

evalscope/models/adapters/server_adapter.py

@@ -1,11 +1,11 @@
 import openai
 from collections import defaultdict
-from inspect import signature
 from openai.types.chat import ChatCompletion, ChatCompletionChunk
 from openai.types.chat.chat_completion import ChatCompletionMessage, Choice
 from typing import List, Optional, Union
 
 from evalscope.utils.logger import get_logger
+from evalscope.utils.utils import get_supported_params
 from .base_adapter import BaseModelAdapter
 
 logger = get_logger()
@@ -31,7 +31,7 @@ class ServerModelAdapter(BaseModelAdapter):
             api_key=api_key,
             base_url=self.api_url,
         )
-        self.supported_params = self._get_supported_params()
+        self.supported_params = get_supported_params(self.client.chat.completions.create)
 
         self.seed = kwargs.get('seed', None)
         self.timeout = kwargs.get('timeout', 60)
@@ -39,11 +39,7 @@ class ServerModelAdapter(BaseModelAdapter):
         self.model_cfg = {'api_url': api_url, 'model_id': model_id, 'api_key': api_key}
         super().__init__(model=None, model_cfg=self.model_cfg, **kwargs)
 
-    def _get_supported_params(self):
-        sig = signature(self.client.chat.completions.create)
-        return list(sig.parameters.keys())
-
-    def predict(self, inputs: List[dict], infer_cfg: dict = None) -> List[dict]:
+    def predict(self, inputs: List[dict], infer_cfg: Optional[dict] = None) -> List[dict]:
         """
         Model prediction func.
 
@@ -65,23 +61,26 @@ class ServerModelAdapter(BaseModelAdapter):
 
     def process_single_input(self, input_item: dict, infer_cfg: dict) -> dict:
         """Process a single input item."""
-        data: list = input_item['data']
-        if isinstance(data[0], tuple):  # for truthful_qa and hellaswag
-            query = '\n'.join(''.join(item) for item in data)
-            system_prompt = input_item.get('system_prompt', None)
+        if input_item.get('messages', None):
+            content = input_item['messages']
         else:
-            query = data[0]
-            system_prompt = input_item.get('system_prompt', None)
-
-        content = self.make_request_content(query, system_prompt)
+            content = self.make_request_content(input_item)
         request_json = self.make_request(content, infer_cfg)
         response = self.send_request(request_json)
         return response
 
-    def make_request_content(self, query: str, system_prompt: Optional[str] = None) -> list:
+    def make_request_content(self, input_item: dict) -> list:
         """
         Make request content for OpenAI API.
         """
+        data: list = input_item['data']
+        if isinstance(data[0], tuple):  # for truthful_qa and hellaswag
+            query = '\n'.join(''.join(item) for item in data)
+            system_prompt = input_item.get('system_prompt', None)
+        else:
+            query = data[0]
+            system_prompt = input_item.get('system_prompt', None)
+
         messages = []
         if system_prompt:
             messages.append({'role': 'system', 'content': system_prompt})
@@ -90,16 +89,9 @@ class ServerModelAdapter(BaseModelAdapter):
 
         return messages
 
-    def make_request(self, content: list, infer_cfg: dict = {}) -> dict:
+    def make_request(self, content: list, infer_cfg: dict) -> dict:
         """Make request to remote API."""
         # Format request JSON according to OpenAI API format
-        from evalscope.config import DEFAULT_GENERATION_CONFIG
-        if infer_cfg == DEFAULT_GENERATION_CONFIG:
-            infer_cfg = {
-                'max_tokens': 2048,
-                'temperature': 0.0,
-            }
-
         request_json = {'model': self.model_id, 'messages': content, **infer_cfg}
 
         if self.timeout:
@@ -137,7 +129,7 @@ class ServerModelAdapter(BaseModelAdapter):
             return response.model_dump(exclude_unset=True)
         except Exception as e:
             logger.error(f'Error when calling remote API: {str(e)}')
-            raise
+            raise e
 
     def _collect_stream_response(self, response_stream: List[ChatCompletionChunk]) -> ChatCompletion:
         collected_chunks = []

evalscope/perf/arguments.py

@@ -3,7 +3,7 @@ import json
 import os
 import sys
 from dataclasses import dataclass, field
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List, Optional, Union
 
 from evalscope.constants import DEFAULT_WORK_DIR
 
@@ -27,8 +27,8 @@ class Arguments:
     no_test_connection: bool = False  # Test the connection before starting the benchmark
 
     # Performance and parallelism
-    number: int = 1000  # Number of requests to be made
-    parallel: int = 1  # Number of parallel requests
+    number: Union[int, List[int]] = 1000  # Number of requests to be made
+    parallel: Union[int, List[int]] = 1  # Number of parallel requests
     rate: int = -1  # Rate limit for requests (default: -1, no limit)
 
     # Logging and debugging
@@ -60,8 +60,8 @@ class Arguments:
     min_tokens: Optional[int] = None  # Minimum number of tokens in the response
     n_choices: Optional[int] = None  # Number of response choices
     seed: Optional[int] = 0  # Random seed for reproducibility
-    stop: Optional[List[str]] = field(default_factory=list)  # Stop sequences for the response
-    stop_token_ids: Optional[List[str]] = field(default_factory=list)  # Stop token IDs for the response
+    stop: Optional[List[str]] = None  # Stop sequences for the response
+    stop_token_ids: Optional[List[str]] = None  # Stop token IDs for the response
     stream: Optional[bool] = True  # Whether to stream the response
     temperature: float = 0.0  # Temperature setting for the response
     top_p: Optional[float] = None  # Top-p (nucleus) sampling setting for the response
@@ -98,6 +98,15 @@ class Arguments:
         if self.apply_chat_template is None:
             self.apply_chat_template = self.url.strip('/').endswith('chat/completions')
 
+        # Set number and parallel to lists if they are integers
+        if isinstance(self.number, int):
+            self.number = [self.number]
+        if isinstance(self.parallel, int):
+            self.parallel = [self.parallel]
+        assert len(self.number) == len(
+            self.parallel
+        ), f'The length of number and parallel should be the same, but got number: {self.number} and parallel: {self.parallel}'  # noqa: E501
+
     def __str__(self):
         return json.dumps(self.to_dict(), indent=4, default=str, ensure_ascii=False)
 
@@ -143,8 +152,8 @@ def add_argument(parser: argparse.ArgumentParser):
     parser.add_argument('--no-test-connection', action='store_false', default=False, help='Do not test the connection before starting the benchmark')  # noqa: E501
 
     # Performance and parallelism
-    parser.add_argument('-n', '--number', type=int, default=1000, help='How many requests to be made')
-    parser.add_argument('--parallel', type=int, default=1, help='Set number of concurrency requests, default 1')
+    parser.add_argument('-n', '--number', type=int, default=1000, nargs='+', help='How many requests to be made')
+    parser.add_argument('--parallel', type=int, default=1, nargs='+', help='Set number of concurrency requests, default 1')  # noqa: E501
     parser.add_argument('--rate', type=int, default=-1, help='Number of requests per second. default None')
 
     # Logging and debugging

evalscope/perf/benchmark.py

@@ -1,11 +1,8 @@
 import asyncio
-import copy
 import json
 import numpy as np
-import os
 import platform
 import sqlite3
-import threading
 import time
 from http import HTTPStatus
 from tqdm import tqdm
@@ -17,8 +14,6 @@ from evalscope.perf.plugin.registry import ApiRegistry, DatasetRegistry
 from evalscope.perf.utils.benchmark_util import BenchmarkData, BenchmarkMetrics
 from evalscope.perf.utils.db_util import create_result_table, get_result_db_path, insert_benchmark_data, summary_result
 from evalscope.perf.utils.handler import add_signal_handlers, exception_handler
-from evalscope.perf.utils.local_server import start_app
-from evalscope.perf.utils.log_utils import init_swanlab, init_wandb
 from evalscope.utils.logger import get_logger
 
 logger = get_logger()
@@ -116,11 +111,6 @@ async def statistic_benchmark_metric(benchmark_data_queue: asyncio.Queue, args:
 
     result_db_path = get_result_db_path(args)
 
-    if args.wandb_api_key:
-        init_wandb(args)
-    if args.swanlab_api_key:
-        init_swanlab(args)
-
     collected_benchmark_data = []
 
     with tqdm(desc='Processing', total=args.number) as pbar:
@@ -170,11 +160,6 @@ async def statistic_benchmark_metric(benchmark_data_queue: asyncio.Queue, args:
 
 @exception_handler
 async def connect_test(args: Arguments) -> bool:
-    if args.api.startswith('local'):
-        #  start local server
-        server = threading.Thread(target=start_app, args=(copy.deepcopy(args), ), daemon=True)
-        server.start()
-
     if (not args.no_test_connection) and (not await test_connection(args)):
         raise TimeoutError('Test connection failed')
 

evalscope/perf/main.py

@@ -1,32 +1,34 @@
 import asyncio
+import copy
 import os
 import platform
+import threading
+import time
 from argparse import Namespace
 
-from evalscope.perf.arguments import Arguments, parse_args
-from evalscope.perf.benchmark import benchmark
-from evalscope.perf.utils.db_util import get_output_path
-from evalscope.perf.utils.handler import add_signal_handlers
+from evalscope.perf.utils.local_server import start_app
+from evalscope.perf.utils.log_utils import init_swanlab, init_wandb
 from evalscope.utils.logger import configure_logging, get_logger
 from evalscope.utils.utils import seed_everything
+from .arguments import Arguments, parse_args
+from .benchmark import benchmark
+from .utils.db_util import get_output_path
+from .utils.handler import add_signal_handlers
+from .utils.rich_display import print_summary
 
 logger = get_logger()
 
 
-def run_perf_benchmark(args):
-    if isinstance(args, dict):
-        args = Arguments(**args)
-    elif isinstance(args, Namespace):
-        args = Arguments.from_args(args)
-
-    if args.seed is not None:
-        seed_everything(args.seed)
+def run_one_benchmark(args: Arguments, output_path: str = None):
+    if isinstance(args.parallel, list):
+        args.parallel = args.parallel[0]
+    if isinstance(args.number, list):
+        args.number = args.number[0]
 
     # Setup logger and output
-    args.outputs_dir = get_output_path(args)
-    configure_logging(args.debug, os.path.join(args.outputs_dir, 'benchmark.log'))
+    args.outputs_dir = output_path
 
-    logger.info('Starting benchmark...')
+    logger.info('Starting benchmark with args: ')
     logger.info(args)
 
     if platform.system() == 'Windows':
@@ -39,6 +41,61 @@ def run_perf_benchmark(args):
     return loop.run_until_complete(benchmark(args))
 
 
+def run_multi_benchmark(args: Arguments, output_path: str = None):
+    results = []
+    number_list = copy.deepcopy(args.number)
+    parallel_list = copy.deepcopy(args.parallel)
+    for i, (number, parallel) in enumerate(zip(number_list, parallel_list)):
+        args.number = number
+        args.parallel = parallel
+        # Set up output path for each run
+        cur_output_path = os.path.join(output_path, f'parallel_{parallel}_number_{number}')
+        os.makedirs(cur_output_path, exist_ok=True)
+        # Start the benchmark
+        metrics_result = run_one_benchmark(args, output_path=cur_output_path)
+        # Save the results
+        results.append(metrics_result)
+        # Sleep between runs to avoid overwhelming the server
+        if i < len(number_list) - 1:
+            logger.info('Sleeping for 5 seconds before the next run...')
+            time.sleep(5)
+    # Analyze results
+    print_summary(results, args.model_id)
+    return results
+
+
+def run_perf_benchmark(args):
+    # Check if args is a dictionary or Namespace
+    if isinstance(args, dict):
+        args = Arguments(**args)
+    elif isinstance(args, Namespace):
+        args = Arguments.from_args(args)
+
+    if args.seed is not None:
+        seed_everything(args.seed)
+
+    # Initialize output directory
+    output_path = get_output_path(args)
+    configure_logging(args.debug, os.path.join(output_path, 'benchmark.log'))
+
+    # Initialize wandb and swanlab
+    if args.wandb_api_key:
+        init_wandb(args)
+    if args.swanlab_api_key:
+        init_swanlab(args)
+
+    # Initialize local server if needed
+    if args.api.startswith('local'):
+        #  start local server
+        server = threading.Thread(target=start_app, args=(copy.deepcopy(args), ), daemon=True)
+        server.start()
+    # Start benchmark
+    if len(args.number) == 1:
+        return run_one_benchmark(args, output_path=output_path)
+    else:
+        return run_multi_benchmark(args, output_path=output_path)
+
+
 if __name__ == '__main__':
     args = Arguments.from_args(parse_args())
     metrics_result, percentile_result = run_perf_benchmark(args)

evalscope/perf/plugin/datasets/custom.py

@@ -22,3 +22,18 @@ class CustomDatasetPlugin(DatasetPluginBase):
                     yield [{'role': 'user', 'content': prompt}]
                 else:
                     yield prompt
+
+
+if __name__ == '__main__':
+    from evalscope.perf.arguments import Arguments
+    from evalscope.perf.main import run_perf_benchmark
+
+    args = Arguments(
+        model='qwen2.5-7b-instruct',
+        url='https://dashscope.aliyuncs.com/compatible-mode/v1/chat/completions',
+        dataset_path='outputs/perf_data.txt',
+        api_key='EMPTY',
+        dataset='custom',
+    )
+
+    run_perf_benchmark(args)

evalscope/perf/utils/benchmark_util.py

@@ -38,7 +38,7 @@ class BenchmarkData:
             self.first_chunk_latency = self.query_latency
             self.n_chunks = 1
             self.n_chunks_time = self.query_latency
-        self.time_per_output_token = self.n_chunks_time / self.completion_tokens
+        self.time_per_output_token = self.n_chunks_time / self.n_chunks
 
     def _calculate_tokens(self, api_plugin):
         self.prompt_tokens, self.completion_tokens = \
@@ -51,6 +51,24 @@ class BenchmarkData:
         self.max_gpu_memory_cost = max(self.max_gpu_memory_cost, total_memory)
 
 
+class Metrics:
+    TIME_TAKEN_FOR_TESTS = 'Time taken for tests (s)'
+    NUMBER_OF_CONCURRENCY = 'Number of concurrency'
+    TOTAL_REQUESTS = 'Total requests'
+    SUCCEED_REQUESTS = 'Succeed requests'
+    FAILED_REQUESTS = 'Failed requests'
+    OUTPUT_TOKEN_THROUGHPUT = 'Output token throughput (tok/s)'
+    TOTAL_TOKEN_THROUGHPUT = 'Total token throughput (tok/s)'
+    REQUEST_THROUGHPUT = 'Request throughput (req/s)'
+    AVERAGE_LATENCY = 'Average latency (s)'
+    AVERAGE_TIME_TO_FIRST_TOKEN = 'Average time to first token (s)'
+    AVERAGE_TIME_PER_OUTPUT_TOKEN = 'Average time per output token (s)'
+    AVERAGE_INPUT_TOKENS_PER_REQUEST = 'Average input tokens per request'
+    AVERAGE_OUTPUT_TOKENS_PER_REQUEST = 'Average output tokens per request'
+    AVERAGE_PACKAGE_LATENCY = 'Average package latency (s)'
+    AVERAGE_PACKAGE_PER_REQUEST = 'Average package per request'
+
+
 @dataclass
 class BenchmarkMetrics:
     concurrency: int = 0
@@ -125,20 +143,20 @@ class BenchmarkMetrics:
 
     def create_message(self, default_ndigits=4):
         message = {
-            'Time taken for tests (s)': round(self.total_time, default_ndigits),
-            'Number of concurrency': self.concurrency,
-            'Total requests': int(self.n_total_queries),
-            'Succeed requests': self.n_succeed_queries,
-            'Failed requests': self.n_failed_queries,
-            'Output token throughput (tok/s)': round(self.avg_output_token_per_seconds, default_ndigits),
-            'Total token throughput (tok/s)': round(self.avg_total_token_per_seconds, default_ndigits),
-            'Request throughput (req/s)': round(self.qps, default_ndigits),
-            'Average latency (s)': round(self.avg_latency, default_ndigits),
-            'Average time to first token (s)': round(self.avg_first_chunk_latency, default_ndigits),
-            'Average time per output token (s)': round(self.avg_time_per_token, default_ndigits),
-            'Average input tokens per request': round(self.avg_prompt_tokens, default_ndigits),
-            'Average output tokens per request': round(self.avg_completion_tokens, default_ndigits),
-            'Average package latency (s)': round(self.avg_chunk_time, default_ndigits),
-            'Average package per request': round(self.n_avg_chunks, default_ndigits),
+            Metrics.TIME_TAKEN_FOR_TESTS: round(self.total_time, default_ndigits),
+            Metrics.NUMBER_OF_CONCURRENCY: self.concurrency,
+            Metrics.TOTAL_REQUESTS: int(self.n_total_queries),
+            Metrics.SUCCEED_REQUESTS: self.n_succeed_queries,
+            Metrics.FAILED_REQUESTS: self.n_failed_queries,
+            Metrics.OUTPUT_TOKEN_THROUGHPUT: round(self.avg_output_token_per_seconds, default_ndigits),
+            Metrics.TOTAL_TOKEN_THROUGHPUT: round(self.avg_total_token_per_seconds, default_ndigits),
+            Metrics.REQUEST_THROUGHPUT: round(self.qps, default_ndigits),
+            Metrics.AVERAGE_LATENCY: round(self.avg_latency, default_ndigits),
+            Metrics.AVERAGE_TIME_TO_FIRST_TOKEN: round(self.avg_first_chunk_latency, default_ndigits),
+            Metrics.AVERAGE_TIME_PER_OUTPUT_TOKEN: round(self.avg_time_per_token, default_ndigits),
+            Metrics.AVERAGE_INPUT_TOKENS_PER_REQUEST: round(self.avg_prompt_tokens, default_ndigits),
+            Metrics.AVERAGE_OUTPUT_TOKENS_PER_REQUEST: round(self.avg_completion_tokens, default_ndigits),
+            Metrics.AVERAGE_PACKAGE_LATENCY: round(self.avg_chunk_time, default_ndigits),
+            Metrics.AVERAGE_PACKAGE_PER_REQUEST: round(self.n_avg_chunks, default_ndigits),
         }
         return message

evalscope/perf/utils/db_util.py

@@ -111,6 +111,18 @@ def get_result_db_path(args: Arguments):
     return result_db_path
 
 
+class PercentileMetrics:
+    TTFT = 'TTFT (s)'
+    ITL = 'ITL (s)'
+    TPOT = 'TPOT (s)'
+    LATENCY = 'Latency (s)'
+    INPUT_TOKENS = 'Input tokens'
+    OUTPUT_TOKENS = 'Output tokens'
+    OUTPUT_THROUGHPUT = 'Output (tok/s)'
+    TOTAL_THROUGHPUT = 'Total (tok/s)'
+    PERCENTILES = 'Percentiles'
+
+
 def calculate_percentiles(data: List[float], percentiles: List[int]) -> Dict[int, float]:
     """
     Calculate the percentiles for a specific list of data.
@@ -157,10 +169,6 @@ def get_percentile_results(result_db_path: str) -> Dict[str, List[float]]:
     with sqlite3.connect(result_db_path) as con:
         rows = con.execute(query_sql).fetchall()
 
-    if len(rows) < len(percentiles):
-        logger.info('Too little data to calculate quantiles!')
-        return {}
-
     # Define index variables for columns
     CHUNK_TIMES_INDEX = 1
     LATENCY_INDEX = 4
@@ -175,24 +183,25 @@ def get_percentile_results(result_db_path: str) -> Dict[str, List[float]]:
         inter_token_latencies_all.extend(inter_token_latencies(row[CHUNK_TIMES_INDEX]))
 
     metrics = {
-        'TTFT (s)': [row[FIRST_CHUNK_LATENCY_INDEX] for row in rows],
-        'ITL (s)':
+        PercentileMetrics.TTFT: [row[FIRST_CHUNK_LATENCY_INDEX] for row in rows],
+        PercentileMetrics.ITL:
         inter_token_latencies_all,
-        'TPOT (s)':
+        PercentileMetrics.TPOT:
         [(row[CHUNK_TIME_INDEX] / row[COMPLETION_TOKENS_INDEX]) if row[COMPLETION_TOKENS_INDEX] > 0 else float('nan')
          for row in rows],
-        'Latency (s)': [row[LATENCY_INDEX] for row in rows],
-        'Input tokens': [row[PROMPT_TOKENS_INDEX] for row in rows],
-        'Output tokens': [row[COMPLETION_TOKENS_INDEX] for row in rows],
-        'Output throughput(tok/s)':
+        PercentileMetrics.LATENCY: [row[LATENCY_INDEX] for row in rows],
+        PercentileMetrics.INPUT_TOKENS: [row[PROMPT_TOKENS_INDEX] for row in rows],
+        PercentileMetrics.OUTPUT_TOKENS: [row[COMPLETION_TOKENS_INDEX] for row in rows],
+        PercentileMetrics.OUTPUT_THROUGHPUT:
         [(row[COMPLETION_TOKENS_INDEX] / row[LATENCY_INDEX]) if row[LATENCY_INDEX] > 0 else float('nan')
          for row in rows],
-        'Total throughput(tok/s)': [((row[PROMPT_TOKENS_INDEX] + row[COMPLETION_TOKENS_INDEX])
-                                     / row[LATENCY_INDEX]) if row[LATENCY_INDEX] > 0 else float('nan') for row in rows]
+        PercentileMetrics.TOTAL_THROUGHPUT: [((row[PROMPT_TOKENS_INDEX] + row[COMPLETION_TOKENS_INDEX])
+                                              / row[LATENCY_INDEX]) if row[LATENCY_INDEX] > 0 else float('nan')
+                                             for row in rows]
     }
 
     # Calculate percentiles for each metric
-    results = {'Percentile': [f'{p}%' for p in percentiles]}
+    results = {PercentileMetrics.PERCENTILES: [f'{p}%' for p in percentiles]}
     for metric_name, data in metrics.items():
         metric_percentiles = calculate_percentiles(data, percentiles)
         results[metric_name] = [metric_percentiles[p] for p in percentiles]
@@ -205,7 +214,6 @@ def summary_result(args: Arguments, metrics: BenchmarkMetrics, result_db_path: s
     write_json_file(args.to_dict(), os.path.join(result_path, 'benchmark_args.json'))
 
     metrics_result = metrics.create_message()
-    metrics_result.update({'Expected number of requests': args.number, 'Result DB path': result_db_path})
     write_json_file(metrics_result, os.path.join(result_path, 'benchmark_summary.json'))
 
     # Print summary in a table
@@ -223,6 +231,8 @@ def summary_result(args: Arguments, metrics: BenchmarkMetrics, result_db_path: s
     if args.dataset.startswith('speed_benchmark'):
         speed_benchmark_result(result_db_path)
 
+    logger.info(f'Save the summary to: {result_path}')
+
     return metrics_result, percentile_result
 
 

evalscope/perf/utils/local_server.py

@@ -96,6 +96,7 @@ def create_app(model, attn_implementation=None) -> FastAPI:
 
 
 def start_app(args: Arguments):
+    logger.info('Starting local server, please wait...')
     if args.api == 'local':
         app = create_app(args.model, args.attn_implementation)
         uvicorn.run(app, host='0.0.0.0', port=args.port, workers=1)

evalscope/perf/utils/log_utils.py

@@ -34,8 +34,15 @@ def init_swanlab(args: Arguments) -> None:
     current_time = datetime.datetime.now().strftime('%Y%m%d_%H%M%S')
     name = args.name if args.name else f'{args.model_id}_{current_time}'
     swanlab.config.update({'framework': '📏evalscope'})
-    swanlab.init(
-        project='perf_benchmark',
-        name=name,
-        config=args.to_dict(),
-        mode='local' if args.swanlab_api_key == 'local' else None)
+    init_kwargs = {
+        'project': os.getenv('SWANLAB_PROJ_NAME', 'perf_benchmark'),
+        'name': name,
+        'config': args.to_dict(),
+        'mode': 'local' if args.swanlab_api_key == 'local' else None
+    }
+
+    workspace = os.getenv('SWANLAB_WORKSPACE')
+    if workspace:
+        init_kwargs['workspace'] = workspace
+
+    swanlab.init(**init_kwargs)