evalscope 0.10.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

evalscope/perf/utils/db_util.py

@@ -2,11 +2,12 @@ import base64
 import json
 import os
 import pickle
+import re
 import sqlite3
 import sys
 from datetime import datetime
 from tabulate import tabulate
-from typing import Dict, List
+from typing import Dict, List, Tuple
 
 from evalscope.perf.arguments import Arguments
 from evalscope.perf.utils.benchmark_util import BenchmarkData, BenchmarkMetrics
@@ -15,6 +16,28 @@ from evalscope.utils.logger import get_logger
 logger = get_logger()
 
 
+class DatabaseColumns:
+    REQUEST = 'request'
+    START_TIME = 'start_time'
+    INTER_TOKEN_LATENCIES = 'inter_token_latencies'
+    SUCCESS = 'success'
+    RESPONSE_MESSAGES = 'response_messages'
+    COMPLETED_TIME = 'completed_time'
+    LATENCY = 'latency'
+    FIRST_CHUNK_LATENCY = 'first_chunk_latency'
+    PROMPT_TOKENS = 'prompt_tokens'
+    COMPLETION_TOKENS = 'completion_tokens'
+    MAX_GPU_MEMORY_COST = 'max_gpu_memory_cost'
+    TIME_PER_OUTPUT_TOKEN = 'time_per_output_token'
+
+
+def load_prompt(prompt_path_or_text):
+    if prompt_path_or_text.startswith('@'):
+        with open(prompt_path_or_text[1:], 'r', encoding='utf-8') as file:
+            return file.read()
+    return prompt_path_or_text
+
+
 def encode_data(data) -> str:
     """Encodes data using base64 and pickle."""
     return base64.b64encode(pickle.dumps(data)).decode('utf-8')
@@ -33,32 +56,34 @@ def transpose_results(data):
 
 
 def create_result_table(cursor):
-    cursor.execute('''CREATE TABLE IF NOT EXISTS result(
-                      request TEXT,
-                      start_time REAL,
-                      chunk_times TEXT,
-                      success INTEGER,
-                      response_messages TEXT,
-                      completed_time REAL,
-                      latency REAL,
-                      first_chunk_latency REAL,
-                      n_chunks INTEGER,
-                      chunk_time REAL,
-                      prompt_tokens INTEGER,
-                      completion_tokens INTEGER,
-                      max_gpu_memory_cost REAL)''')
+    cursor.execute(
+        f'''CREATE TABLE IF NOT EXISTS result(
+                      {DatabaseColumns.REQUEST} TEXT,
+                      {DatabaseColumns.START_TIME} REAL,
+                      {DatabaseColumns.INTER_TOKEN_LATENCIES} TEXT,
+                      {DatabaseColumns.SUCCESS} INTEGER,
+                      {DatabaseColumns.RESPONSE_MESSAGES} TEXT,
+                      {DatabaseColumns.COMPLETED_TIME} REAL,
+                      {DatabaseColumns.LATENCY} REAL,
+                      {DatabaseColumns.FIRST_CHUNK_LATENCY} REAL,
+                      {DatabaseColumns.PROMPT_TOKENS} INTEGER,
+                      {DatabaseColumns.COMPLETION_TOKENS} INTEGER,
+                      {DatabaseColumns.MAX_GPU_MEMORY_COST} REAL,
+                      {DatabaseColumns.TIME_PER_OUTPUT_TOKEN} REAL
+                   )'''
+    )
 
 
 def insert_benchmark_data(cursor: sqlite3.Cursor, benchmark_data: BenchmarkData):
-    request = encode_data(benchmark_data.request)
-    chunk_times = json.dumps(benchmark_data.chunk_times)
+    request = benchmark_data.request
+    inter_token_latencies = json.dumps(benchmark_data.inter_chunk_latency)
     response_messages = encode_data(benchmark_data.response_messages)
 
     # Columns common to both success and failure cases
     common_columns = (
         request,
         benchmark_data.start_time,
-        chunk_times,
+        inter_token_latencies,
         benchmark_data.success,
         response_messages,
         benchmark_data.completed_time,
@@ -67,23 +92,21 @@ def insert_benchmark_data(cursor: sqlite3.Cursor, benchmark_data: BenchmarkData)
     if benchmark_data.success:
         # Add additional columns for success case
         additional_columns = (
-            benchmark_data.query_latency,
-            benchmark_data.first_chunk_latency,
-            benchmark_data.n_chunks,
-            benchmark_data.n_chunks_time,
-            benchmark_data.prompt_tokens,
-            benchmark_data.completion_tokens,
-            benchmark_data.max_gpu_memory_cost,
+            benchmark_data.query_latency, benchmark_data.first_chunk_latency, benchmark_data.prompt_tokens,
+            benchmark_data.completion_tokens, benchmark_data.max_gpu_memory_cost, benchmark_data.time_per_output_token
         )
-        query = """INSERT INTO result(
-                      request, start_time, chunk_times, success, response_messages,
-                      completed_time, latency, first_chunk_latency,
-                      n_chunks, chunk_time, prompt_tokens, completion_tokens, max_gpu_memory_cost
-                   ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"""
+        query = f"""INSERT INTO result(
+                      {DatabaseColumns.REQUEST}, {DatabaseColumns.START_TIME}, {DatabaseColumns.INTER_TOKEN_LATENCIES},
+                      {DatabaseColumns.SUCCESS}, {DatabaseColumns.RESPONSE_MESSAGES}, {DatabaseColumns.COMPLETED_TIME},
+                      {DatabaseColumns.LATENCY}, {DatabaseColumns.FIRST_CHUNK_LATENCY}, {DatabaseColumns.PROMPT_TOKENS},
+                      {DatabaseColumns.COMPLETION_TOKENS}, {DatabaseColumns.MAX_GPU_MEMORY_COST},
+                      {DatabaseColumns.TIME_PER_OUTPUT_TOKEN}
+                   ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"""
         cursor.execute(query, common_columns + additional_columns)
     else:
-        query = """INSERT INTO result(
-                      request, start_time, chunk_times, success, response_messages, completed_time
+        query = f"""INSERT INTO result(
+                      {DatabaseColumns.REQUEST}, {DatabaseColumns.START_TIME}, {DatabaseColumns.INTER_TOKEN_LATENCIES},
+                      {DatabaseColumns.SUCCESS}, {DatabaseColumns.RESPONSE_MESSAGES}, {DatabaseColumns.COMPLETED_TIME}
                    ) VALUES (?, ?, ?, ?, ?, ?)"""
         cursor.execute(query, common_columns)
 
@@ -91,6 +114,8 @@ def insert_benchmark_data(cursor: sqlite3.Cursor, benchmark_data: BenchmarkData)
 def get_output_path(args: Arguments) -> str:
     current_time = datetime.now().strftime('%Y%m%d_%H%M%S')
     output_path = os.path.join(args.outputs_dir, current_time, f'{args.name or args.model_id}')
+    # Filter illegal characters
+    output_path = re.sub(r'[<>:"|?*]', '_', output_path)
     if not os.path.exists(output_path):
         os.makedirs(output_path, exist_ok=True)
     logger.info(f'Save the result to: {output_path}')
@@ -102,12 +127,24 @@ def get_result_db_path(args: Arguments):
 
     logger.info(f'Save the data base to: {result_db_path}')
     if os.path.exists(result_db_path):
-        logger.warning('The db file exists, delete it and start again!.')
+        logger.error(f'The db file {result_db_path} exists, delete it and start again!.')
         sys.exit(1)
 
     return result_db_path
 
 
+class PercentileMetrics:
+    TTFT = 'TTFT (s)'
+    ITL = 'ITL (s)'
+    TPOT = 'TPOT (s)'
+    LATENCY = 'Latency (s)'
+    INPUT_TOKENS = 'Input tokens'
+    OUTPUT_TOKENS = 'Output tokens'
+    OUTPUT_THROUGHPUT = 'Output (tok/s)'
+    TOTAL_THROUGHPUT = 'Total (tok/s)'
+    PERCENTILES = 'Percentiles'
+
+
 def calculate_percentiles(data: List[float], percentiles: List[int]) -> Dict[int, float]:
     """
     Calculate the percentiles for a specific list of data.
@@ -136,54 +173,51 @@ def get_percentile_results(result_db_path: str) -> Dict[str, List[float]]:
     :param result_db_path: Path to the SQLite database file.
     :return: Dictionary of percentiles for various metrics.
     """
-
-    def inter_token_latencies(chunk_times_json: str) -> List[float]:
-        try:
-            chunk_times = json.loads(chunk_times_json)
-            return [t2 - t1 for t1, t2 in zip(chunk_times[:-1], chunk_times[1:])]
-        except (json.JSONDecodeError, TypeError) as e:
-            logger.error(f'Error parsing chunk times: {e}')
-            return []
-
-    query_sql = ('SELECT start_time, chunk_times, success, completed_time, latency, first_chunk_latency, '
-                 'n_chunks, chunk_time, prompt_tokens, completion_tokens '
-                 'FROM result WHERE success=1')
+    query_sql = f'''SELECT {DatabaseColumns.START_TIME}, {DatabaseColumns.INTER_TOKEN_LATENCIES}, {DatabaseColumns.SUCCESS},
+                    {DatabaseColumns.COMPLETED_TIME}, {DatabaseColumns.LATENCY}, {DatabaseColumns.FIRST_CHUNK_LATENCY},
+                    {DatabaseColumns.PROMPT_TOKENS},
+                    {DatabaseColumns.COMPLETION_TOKENS}, {DatabaseColumns.TIME_PER_OUTPUT_TOKEN}
+                    FROM result WHERE {DatabaseColumns.SUCCESS}=1'''  # noqa: E501
 
     percentiles = [10, 25, 50, 66, 75, 80, 90, 95, 98, 99]
 
     with sqlite3.connect(result_db_path) as con:
-        rows = con.execute(query_sql).fetchall()
-
-    if len(rows) < len(percentiles):
-        logger.info('Too little data to calculate quantiles!')
-        return {}
+        cursor = con.cursor()
+        cursor.execute(query_sql)
+        columns = [description[0] for description in cursor.description]
+        rows = cursor.fetchall()
 
-    # Define index variables for columns
-    CHUNK_TIMES_INDEX = 1
-    LATENCY_INDEX = 4
-    FIRST_CHUNK_LATENCY_INDEX = 5
-    PROMPT_TOKENS_INDEX = 8
-    COMPLETION_TOKENS_INDEX = 9
+    # Create column index mapping
+    col_indices = {col: idx for idx, col in enumerate(columns)}
 
     # Prepare data for each metric
     inter_token_latencies_all = []
     for row in rows:
-        inter_token_latencies_all.extend(inter_token_latencies(row[CHUNK_TIMES_INDEX]))
+        try:
+            itl = json.loads(row[col_indices[DatabaseColumns.INTER_TOKEN_LATENCIES]]) or []
+            inter_token_latencies_all.extend(itl)
+        except (json.JSONDecodeError, TypeError) as e:
+            logger.error(f'Error parsing inter token latencies: {e}')
 
     metrics = {
-        'TTFT (s)': [row[FIRST_CHUNK_LATENCY_INDEX] for row in rows],
-        'TPOT (s)':
+        PercentileMetrics.TTFT: [row[col_indices[DatabaseColumns.FIRST_CHUNK_LATENCY]] for row in rows],
+        PercentileMetrics.ITL:
         inter_token_latencies_all,
-        'Latency (s)': [row[LATENCY_INDEX] for row in rows],
-        'Input tokens': [row[PROMPT_TOKENS_INDEX] for row in rows],
-        'Output tokens': [row[COMPLETION_TOKENS_INDEX] for row in rows],
-        'Throughput(tokens/s)':
-        [(row[COMPLETION_TOKENS_INDEX] / row[LATENCY_INDEX]) if row[LATENCY_INDEX] > 0 else float('nan')
-         for row in rows]
+        PercentileMetrics.TPOT: [row[col_indices[DatabaseColumns.TIME_PER_OUTPUT_TOKEN]] for row in rows],
+        PercentileMetrics.LATENCY: [row[col_indices[DatabaseColumns.LATENCY]] for row in rows],
+        PercentileMetrics.INPUT_TOKENS: [row[col_indices[DatabaseColumns.PROMPT_TOKENS]] for row in rows],
+        PercentileMetrics.OUTPUT_TOKENS: [row[col_indices[DatabaseColumns.COMPLETION_TOKENS]] for row in rows],
+        PercentileMetrics.OUTPUT_THROUGHPUT:
+        [(row[col_indices[DatabaseColumns.COMPLETION_TOKENS]] / row[col_indices[DatabaseColumns.LATENCY]])
+         if row[col_indices[DatabaseColumns.LATENCY]] > 0 else float('nan') for row in rows],
+        PercentileMetrics.TOTAL_THROUGHPUT:
+        [((row[col_indices[DatabaseColumns.PROMPT_TOKENS]] + row[col_indices[DatabaseColumns.COMPLETION_TOKENS]])
+          / row[col_indices[DatabaseColumns.LATENCY]])
+         if row[col_indices[DatabaseColumns.LATENCY]] > 0 else float('nan') for row in rows]
     }
 
     # Calculate percentiles for each metric
-    results = {'Percentile': [f'{p}%' for p in percentiles]}
+    results = {PercentileMetrics.PERCENTILES: [f'{p}%' for p in percentiles]}
     for metric_name, data in metrics.items():
         metric_percentiles = calculate_percentiles(data, percentiles)
         results[metric_name] = [metric_percentiles[p] for p in percentiles]
@@ -191,16 +225,15 @@ def get_percentile_results(result_db_path: str) -> Dict[str, List[float]]:
     return results
 
 
-def summary_result(args: Arguments, metrics: BenchmarkMetrics, expected_number_of_queries: int, result_db_path: str):
+def summary_result(args: Arguments, metrics: BenchmarkMetrics, result_db_path: str) -> Tuple[Dict, Dict]:
     result_path = os.path.dirname(result_db_path)
     write_json_file(args.to_dict(), os.path.join(result_path, 'benchmark_args.json'))
 
-    data = metrics.create_message()
-    data.update({'Expected number of requests': expected_number_of_queries, 'Result DB path': result_db_path})
-    write_json_file(data, os.path.join(result_path, 'benchmark_summary.json'))
+    metrics_result = metrics.create_message()
+    write_json_file(metrics_result, os.path.join(result_path, 'benchmark_summary.json'))
 
     # Print summary in a table
-    table = tabulate(list(data.items()), headers=['Key', 'Value'], tablefmt='grid')
+    table = tabulate(list(metrics_result.items()), headers=['Key', 'Value'], tablefmt='grid')
     logger.info('\nBenchmarking summary:\n' + table)
 
     # Get percentile results
@@ -214,20 +247,24 @@ def summary_result(args: Arguments, metrics: BenchmarkMetrics, expected_number_o
     if args.dataset.startswith('speed_benchmark'):
         speed_benchmark_result(result_db_path)
 
+    logger.info(f'Save the summary to: {result_path}')
+
+    return metrics_result, percentile_result
+
 
 def speed_benchmark_result(result_db_path: str):
-    query_sql = """
+    query_sql = f"""
         SELECT
-            prompt_tokens,
-            ROUND(AVG(completion_tokens / latency), 2) AS avg_completion_token_per_second,
-            ROUND(AVG(max_gpu_memory_cost), 2)
+            {DatabaseColumns.PROMPT_TOKENS},
+            ROUND(AVG({DatabaseColumns.COMPLETION_TOKENS} / {DatabaseColumns.LATENCY}), 2) AS avg_completion_token_per_second,
+            ROUND(AVG({DatabaseColumns.MAX_GPU_MEMORY_COST}), 2)
         FROM
             result
         WHERE
-            success = 1 AND latency > 0
+            {DatabaseColumns.SUCCESS} = 1 AND {DatabaseColumns.LATENCY} > 0
         GROUP BY
-            prompt_tokens
-    """
+            {DatabaseColumns.PROMPT_TOKENS}
+    """  # noqa: E501
 
     with sqlite3.connect(result_db_path) as con:
         cursor = con.cursor()

evalscope/perf/utils/local_server.py

@@ -1,68 +1,28 @@
 import os
 import subprocess
-import torch
 import uvicorn
 from contextlib import asynccontextmanager
-from dataclasses import dataclass
 from fastapi import FastAPI
 from fastapi.middleware.cors import CORSMiddleware
 from sse_starlette.sse import EventSourceResponse
 
 from evalscope.perf.arguments import Arguments
 from evalscope.utils.chat_service import ChatCompletionRequest, ChatService, ModelList, TextCompletionRequest
+from evalscope.utils.import_utils import check_import
 from evalscope.utils.logger import get_logger
 
 logger = get_logger()
 
 
-@dataclass
-class ServerSentEvent(object):
-
-    def __init__(self, data='', event=None, id=None, retry=None):
-        self.data = data
-        self.event = event
-        self.id = id
-        self.retry = retry
-
-    @classmethod
-    def decode(cls, line):
-        """Decode line to ServerSentEvent
-
-
-        Args:
-            line (str): The line.
-
-        Return:
-            ServerSentEvent (obj:`ServerSentEvent`): The ServerSentEvent object.
-
-        """
-        if not line:
-            return None
-        sse_msg = cls()
-        # format data:xxx
-        field_type, _, field_value = line.partition(':')
-        if field_value.startswith(' '):  # compatible with openai api
-            field_value = field_value[1:]
-        if field_type == 'event':
-            sse_msg.event = field_value
-        elif field_type == 'data':
-            field_value = field_value.rstrip()
-            sse_msg.data = field_value
-        elif field_type == 'id':
-            sse_msg.id = field_value
-        elif field_type == 'retry':
-            sse_msg.retry = field_value
-        else:
-            pass
-
-        return sse_msg
-
-
 @asynccontextmanager
 async def lifespan(app: FastAPI):
     yield
-    if torch.cuda.is_available():
-        torch.cuda.empty_cache()
+    try:
+        import torch
+        if torch.cuda.is_available():
+            torch.cuda.empty_cache()
+    except ImportError:
+        pass
 
 
 def create_app(model, attn_implementation=None) -> FastAPI:
@@ -96,11 +56,16 @@ def create_app(model, attn_implementation=None) -> FastAPI:
 
 
 def start_app(args: Arguments):
+    logger.info('Starting local server, please wait...')
     if args.api == 'local':
+        check_import('torch', 'torch', raise_error=True)
+
         app = create_app(args.model, args.attn_implementation)
         uvicorn.run(app, host='0.0.0.0', port=args.port, workers=1)
 
     elif args.api == 'local_vllm':
+        import torch
+
         os.environ['VLLM_USE_MODELSCOPE'] = 'True'
         os.environ['VLLM_ALLOW_LONG_MAX_MODEL_LEN'] = '1'
         os.environ['VLLM_WORKER_MULTIPROC_METHOD'] = 'spawn'

evalscope/perf/utils/log_utils.py

@@ -0,0 +1,63 @@
+import os
+
+from evalscope.perf.arguments import Arguments
+
+
+def init_wandb(args: Arguments) -> None:
+    """
+    Initialize WandB for logging.
+    """
+    # Initialize wandb if the api key is provided
+    import datetime
+    try:
+        import wandb
+    except ImportError:
+        raise RuntimeError('Cannot import wandb. Please install it with command: \n pip install wandb')
+    os.environ['WANDB_SILENT'] = 'true'
+    os.environ['WANDB_DIR'] = args.outputs_dir
+    current_time = datetime.datetime.now().strftime('%Y%m%d_%H%M%S')
+    name = args.name if args.name else f'{args.model_id}_{current_time}'
+
+    # Remove sensitive information from logging config
+    logging_config = args.to_dict()
+    logging_config.pop('api_key', None)
+    logging_config.pop('wandb_api_key', None)
+
+    if args.wandb_api_key is not None:
+        wandb.login(key=args.wandb_api_key)
+    wandb.init(project='perf_benchmark', name=name, config=logging_config)
+
+
+def init_swanlab(args: Arguments) -> None:
+    """
+    Initialize SwanLab for logging.
+    """
+    import datetime
+    try:
+        import swanlab
+    except ImportError:
+        raise RuntimeError('Cannot import swanlab. Please install it with command: \n pip install swanlab')
+    os.environ['SWANLAB_LOG_DIR'] = args.outputs_dir
+    current_time = datetime.datetime.now().strftime('%Y%m%d_%H%M%S')
+    name = args.name if args.name else f'{args.model_id}_{current_time}'
+    swanlab.config.update({'framework': '📏evalscope'})
+
+    # Remove sensitive information from logging config
+    logging_config = args.to_dict()
+    logging_config.pop('api_key', None)
+    logging_config.pop('swanlab_api_key', None)
+
+    init_kwargs = {
+        'project': os.getenv('SWANLAB_PROJ_NAME', 'perf_benchmark'),
+        'name': name,
+        'config': logging_config,
+        'mode': 'local' if args.swanlab_api_key == 'local' else None
+    }
+
+    workspace = os.getenv('SWANLAB_WORKSPACE')
+    if workspace:
+        init_kwargs['workspace'] = workspace
+
+    if isinstance(args.swanlab_api_key, str) and not args.swanlab_api_key == 'local':
+        swanlab.login(api_key=args.swanlab_api_key)
+    swanlab.init(**init_kwargs)

evalscope/perf/utils/rich_display.py

@@ -0,0 +1,192 @@
+# the following code is largely adapted from https://github.com/lework/llm-benchmark
+
+import numpy as np
+from rich.console import Console
+from rich.panel import Panel
+from rich.style import Style
+from rich.table import Table
+from rich.text import Text
+
+from evalscope.utils.logger import get_logger
+from .benchmark_util import Metrics
+from .db_util import PercentileMetrics
+
+logger = get_logger()
+
+
+def analyze_results(all_results):
+    """Analyze all test results and generate a summary report"""
+    summary = []
+    total_tokens = 0
+    total_time = 0
+
+    for result in all_results:
+        total_metrics = result[0]
+        percentile_metrics = result[1]
+        percentiles = percentile_metrics[PercentileMetrics.PERCENTILES]
+        try:
+            concurrency = total_metrics.get(Metrics.NUMBER_OF_CONCURRENCY, 0)
+            rps = total_metrics.get(Metrics.REQUEST_THROUGHPUT, 0)
+            avg_latency = total_metrics.get(Metrics.AVERAGE_LATENCY, 0)
+            p99_latency = percentile_metrics.get(PercentileMetrics.LATENCY)[percentiles.index('99%')]
+            avg_tps = total_metrics.get(Metrics.OUTPUT_TOKEN_THROUGHPUT, 0)
+            avg_ttft = total_metrics.get(Metrics.AVERAGE_TIME_TO_FIRST_TOKEN, 0)
+            p99_ttft = percentile_metrics.get(PercentileMetrics.TTFT)[percentiles.index('99%')]
+            success_rate = (
+                total_metrics.get(Metrics.SUCCEED_REQUESTS, 0) / total_metrics.get(Metrics.TOTAL_REQUESTS, 1)
+            ) * 100
+            avg_tpot = total_metrics.get(Metrics.AVERAGE_TIME_PER_OUTPUT_TOKEN, 0)
+            p99_tpot = percentile_metrics.get(PercentileMetrics.TPOT)[percentiles.index('99%')]
+
+            # Ensure all values are valid numbers
+            if any(x is None for x in [concurrency, rps, avg_latency, p99_latency, avg_tps, avg_ttft]):
+                logger.warning(f'Warning: Test results for concurrency {concurrency} contain invalid data, skipped')
+                continue
+
+            summary.append([
+                concurrency,
+                f'{rps:.2f}' if rps is not None else 'N/A',
+                f'{avg_latency:.3f}' if avg_latency is not None else 'N/A',
+                f'{p99_latency:.3f}' if p99_latency is not None else 'N/A',
+                f'{avg_tps:.2f}' if avg_tps is not None else 'N/A',
+                f'{avg_ttft:.3f}' if avg_ttft is not None else 'N/A',
+                f'{success_rate:.1f}%' if success_rate is not None else 'N/A',
+                f'{p99_ttft:.3f}' if p99_ttft is not None else 'N/A',
+                f'{avg_tpot:.3f}' if avg_tpot is not None else 'N/A',
+                f'{p99_tpot:.3f}' if p99_tpot is not None else 'N/A',
+            ])
+
+            total_tokens += total_metrics.get(Metrics.AVERAGE_OUTPUT_TOKENS_PER_REQUEST,
+                                              0) * total_metrics.get(Metrics.SUCCEED_REQUESTS, 0)
+            total_time += total_metrics.get(Metrics.TIME_TAKEN_FOR_TESTS, 0)
+        except Exception as e:
+            logger.warning(
+                f"Warning: Error processing results for concurrency {result.get('concurrency', 'unknown')}: {str(e)}"
+            )
+            continue
+
+    if not summary:
+        logger.warning('Error: No valid test result data')
+        return [], 0, 0
+
+    return summary, total_tokens, total_time
+
+
+def print_summary(all_results, model_name):
+    """Print test results summary"""
+    summary, total_tokens, total_time = analyze_results(all_results)
+
+    if not summary:
+        logger.warning('No available test result data to display')
+        return
+
+    console = Console(width=100)  # Set fixed width
+
+    # Create title panel
+    title = Text('Performance Test Summary Report', style='bold')
+    console.print(Panel(title, width=60))
+
+    # Print basic information
+    basic_info = Table(show_header=False, width=60)
+    basic_info.add_column('Name', style='cyan', width=25)
+    basic_info.add_column('Value', style='green', width=35)
+
+    basic_info.add_row('Model', model_name)
+    basic_info.add_row('Total Generated', f'{total_tokens:,} tokens')
+    basic_info.add_row('Total Test Time', f'{total_time:.2f} seconds')
+    basic_info.add_row('Avg Output Rate', f'{total_tokens / total_time:.2f} tokens/sec')
+
+    console.print('\nBasic Information:')
+    console.print(basic_info)
+
+    # Create detailed performance metrics table
+    table = Table(
+        title='Detailed Performance Metrics',
+        show_header=True,
+        header_style='bold cyan',
+        border_style='blue',
+        width=100,  # Set total table width
+        pad_edge=False,  # Reduce edge padding
+        min_width=60,  # Minimum width
+    )
+
+    # Add columns (set fixed column widths)
+    table.add_column('Conc.', justify='right', style='cyan')
+    table.add_column('RPS', justify='right')
+    table.add_column('Avg Lat.(s)', justify='right')
+    table.add_column('P99 Lat.(s)', justify='right')
+    table.add_column('Gen. toks/s', justify='right')
+    table.add_column('Avg TTFT(s)', justify='right')
+    table.add_column('P99 TTFT(s)', justify='right')
+    table.add_column('Avg TPOT(s)', justify='right')
+    table.add_column('P99 TPOT(s)', justify='right')
+    table.add_column('Success Rate', justify='right', style='green')
+
+    # Add data rows
+    for row in summary:
+        try:
+            # Set row style based on success rate
+            success_rate = float(row[6].rstrip('%'))
+            row_style = 'green' if success_rate >= 95 else 'yellow' if success_rate >= 80 else 'red'
+
+            table.add_row(
+                str(row[0]),  # Concurrency
+                f'{float(row[1]):.2f}',  # RPS
+                f'{float(row[2]):.3f}',  # Average Latency
+                f'{float(row[3]):.3f}',  # P99 Latency
+                f'{float(row[4]):.2f}',  # Average TPS
+                f'{float(row[5]):.3f}',  # First Token Latency
+                f'{float(row[7]):.3f}',  # P99 TTFT
+                f'{float(row[8]):.3f}',  # Average TPOT
+                f'{float(row[9]):.3f}',  # P99 TPOT
+                row[6],  # Success Rate
+                style=row_style
+            )
+        except ValueError as e:
+            console.print(f'Warning: Error processing row data: {str(e)}', style='bold red')
+            continue
+
+    console.print('\n')
+    console.print(table)
+
+    # Calculate and display best performance configuration
+    try:
+        best_rps_idx = np.argmax([float(row[1]) if row[1] != 'N/A' else -1 for row in summary])
+        best_latency_idx = np.argmin([float(row[2]) if row[2] != 'N/A' else float('inf') for row in summary])
+
+        perf_info = Table(title='Best Performance Configuration', show_header=False, box=None, width=60)
+        perf_info.add_column('Metric', style='cyan', width=20)
+        perf_info.add_column('Value', style='green', width=40)
+
+        perf_info.add_row('Highest RPS', f'Concurrency {summary[best_rps_idx][0]} ({summary[best_rps_idx][1]} req/sec)')
+        perf_info.add_row(
+            'Lowest Latency', f'Concurrency {summary[best_latency_idx][0]} ({summary[best_latency_idx][2]} seconds)'
+        )
+
+        console.print('\n')
+        console.print(perf_info)
+
+        # Performance recommendations
+        recommendations = []
+        if best_rps_idx == len(summary) - 1:
+            recommendations.append(
+                'The system seems not to have reached its performance bottleneck, try higher concurrency'
+            )
+        elif best_rps_idx == 0:
+            recommendations.append('Consider lowering concurrency, current load may be too high')
+        else:
+            recommendations.append(f'Optimal concurrency range is around {summary[best_rps_idx][0]}')
+
+        success_rate = float(summary[-1][6][:-1])
+        if success_rate < 95:
+            recommendations.append(
+                'Success rate is low at high concurrency, check system resources or reduce concurrency'
+            )
+
+        recommend_text = Text('\nPerformance Recommendations:', style='bold cyan')
+        console.print(recommend_text)
+        for rec in recommendations:
+            console.print(f'• {rec}', style='yellow')
+
+    except Exception as e:
+        console.print(f'Warning: Error generating performance analysis: {str(e)}', style='bold red')