evalscope 0.10.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

evalscope/perf/plugin/datasets/speed_benchmark.py

@@ -3,6 +3,9 @@ from typing import Dict, Iterator, List, Tuple
 from evalscope.perf.arguments import Arguments
 from evalscope.perf.plugin.datasets.base import DatasetPluginBase
 from evalscope.perf.plugin.registry import register_dataset
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
 
 
 @register_dataset('speed_benchmark')
@@ -18,6 +21,14 @@ class SpeedBenchmarkDatasetPlugin(DatasetPluginBase):
     def __init__(self, query_parameters: Arguments):
         super().__init__(query_parameters)
 
+        url = self.query_parameters.url
+        if url.endswith('v1/chat/completions'):
+            logger.warning(
+                'The API URL is not set correctly for `speed_benchmark`. Using `v1/completions` instead of `v1/chat/completions` by system.'  # noqa
+            )
+            url = url.replace('v1/chat/completions', 'v1/completions')
+            self.query_parameters.url = url
+
     def build_messages(self) -> Iterator[List[Dict]]:
         for input_len in self.INPUT_LENGTH:
             for _ in range(self.REPEAT):

evalscope/perf/plugin/registry.py

@@ -1,23 +1,25 @@
-from typing import Any, List, Type, Union
+from typing import TYPE_CHECKING, Any, List, Type, Union
 
+if TYPE_CHECKING:
+    from .api import ApiPluginBase
+    from .datasets import DatasetPluginBase
 
-class PluginRegistry:
-
-    def __init__(self):
-        self._registry = {}
 
-    def register(self, name, cls):
-        self._registry[name] = cls
-        return cls
+class PluginRegistry:
+    _registry = {}
 
-    def get_class(self, name):
-        return self._registry[name]
+    @classmethod
+    def register(cls, name, plugin_cls):
+        cls._registry[name] = plugin_cls
+        return plugin_cls
 
-    def all_classes(self):
-        return list(self._registry.keys())
+    @classmethod
+    def get_class(cls, name):
+        return cls._registry[name]
 
-    def __call__(self, name: str) -> Any:
-        return self.get_class(name)
+    @classmethod
+    def all_classes(cls):
+        return list(cls._registry.keys())
 
 
 def register_dataset(name: Union[str, List[str]]):
@@ -50,5 +52,23 @@ def register_api(name: Union[str, List[str]]):
     return class_decorator
 
 
-DatasetRegistry = PluginRegistry()
-ApiRegistry = PluginRegistry()
+class DatasetRegistry(PluginRegistry):
+    """Registry for dataset plugins."""
+    _registry = {}
+
+    @classmethod
+    def get_class(cls, name: str) -> Type['DatasetPluginBase']:
+        if name not in cls._registry:
+            raise ValueError(f"Dataset plugin '{name}' is not registered.")
+        return cls._registry[name]
+
+
+class ApiRegistry(PluginRegistry):
+    """Registry for API plugins."""
+    _registry = {}
+
+    @classmethod
+    def get_class(cls, name: str) -> Type['ApiPluginBase']:
+        if name not in cls._registry:
+            raise ValueError(f"API plugin '{name}' is not registered.")
+        return cls._registry[name]

evalscope/perf/utils/analysis_result.py

@@ -3,27 +3,28 @@ import json
 import pickle
 import sqlite3
 
-result_db_path = '/mnt/data/data/user/maoyunlin.myl/eval-scope/outputs/qwen2.5_benchmark_20241111_160543.db'
-con = sqlite3.connect(result_db_path)
-query_sql = "SELECT request, response_messages, prompt_tokens, completion_tokens \
-                FROM result WHERE success='1'"
+db_path = 'your db path'
+conn = sqlite3.connect(db_path)
+cursor = conn.cursor()
 
-# how to save base64.b64encode(pickle.dumps(benchmark_data["request"])).decode("ascii"),
-with con:
-    rows = con.execute(query_sql).fetchall()
-    if len(rows) > 0:
-        for row in rows:
-            request = row[0]
-            responses = row[1]
-            request = base64.b64decode(request)
-            request = pickle.loads(request)
-            responses = base64.b64decode(responses)
-            responses = pickle.loads(responses)
-            response_content = ''
-            for response in responses:
-                response = json.loads(response)
-                if not response['choices']:
-                    continue
-                response_content += response['choices'][0]['delta']['content']
-            print('prompt: %s, tokens: %s, completion: %s, tokens: %s' %
-                  (request['messages'][0]['content'], row[2], response_content, row[3]))
+# 获取列名
+cursor.execute('PRAGMA table_info(result)')
+columns = [info[1] for info in cursor.fetchall()]
+print('列名：', columns)
+
+cursor.execute('SELECT * FROM result WHERE success=1 AND first_chunk_latency > 1')
+rows = cursor.fetchall()
+print(f'len(rows): {len(rows)}')
+
+for row in rows:
+    row_dict = dict(zip(columns, row))
+    # 解码request
+    row_dict['request'] = pickle.loads(base64.b64decode(row_dict['request']))
+    # 解码response_messages
+    row_dict['response_messages'] = pickle.loads(base64.b64decode(row_dict['response_messages']))
+    # print(row_dict)
+    print(
+        f"request_id: {json.loads(row_dict['response_messages'][0])['id']}, first_chunk_latency: {row_dict['first_chunk_latency']}"  # noqa: E501
+    )
+    # 如果只想看一个可以break
+    # break

evalscope/perf/utils/benchmark_util.py

@@ -1,8 +1,7 @@
-import time
-import torch
 from dataclasses import dataclass, field
 from typing import Any, List, Optional, Tuple
 
+from evalscope.utils.import_utils import check_import
 from evalscope.utils.logger import get_logger
 
 logger = get_logger()
@@ -10,8 +9,8 @@ logger = get_logger()
 
 @dataclass
 class BenchmarkData:
-    request: Any = None
-    start_time: float = field(default_factory=time.perf_counter)
+    request: str = None  # json serialized request body
+    start_time: float = 0.0
     completed_time: float = 0.0
     chunk_times: List[float] = field(default_factory=list)
     success: bool = False
@@ -20,33 +19,55 @@ class BenchmarkData:
     # late init
     query_latency: float = 0.0
     first_chunk_latency: float = 0.0
-    n_chunks: int = 0
-    n_chunks_time: float = 0.0
     max_gpu_memory_cost = 0
+    time_per_output_token: float = 0.0
+    inter_chunk_latency: List[float] = field(default_factory=list)
 
-    prompt_tokens = None
-    completion_tokens = None
-
-    def _calculate_query_stream_metric(self) -> Tuple[float, int, float]:
-        self.query_latency = self.completed_time - self.start_time
-        if len(self.chunk_times) > 1:
-            self.first_chunk_latency = self.chunk_times[0] - self.start_time
-            self.n_chunks = len(self.chunk_times) - 2
-            self.n_chunks_time = self.chunk_times[-2] - self.chunk_times[0]
-        else:
-            self.first_chunk_latency = self.query_latency
-            self.n_chunks = 1
-            self.n_chunks_time = self.query_latency
+    # response content
+    generated_text: str = ''
+    error: Optional[str] = None
+    prompt_tokens: Optional[int] = None
+    completion_tokens: Optional[int] = None
 
     def _calculate_tokens(self, api_plugin):
-        self.prompt_tokens, self.completion_tokens = \
-            api_plugin.parse_responses(self.response_messages, request=self.request)
+        if self.prompt_tokens is None or self.completion_tokens is None:
+            self.prompt_tokens, self.completion_tokens = api_plugin.parse_responses(
+                self.response_messages, request=self.request
+            )
+
+        # Calculate time per output token
+        if self.completion_tokens and self.completion_tokens > 1:
+            # tpot = (latency - ttft) / (output_len - 1)
+            self.time_per_output_token = (self.query_latency - self.first_chunk_latency) / (self.completion_tokens - 1)
+
+        # Ensure inter-chunk latency is available (compute from chunk_times if needed)
+        if not self.inter_chunk_latency and self.chunk_times:
+            self.inter_chunk_latency = [t2 - t1 for t1, t2 in zip(self.chunk_times[:-1], self.chunk_times[1:])]
 
     def update_gpu_usage(self):
-        total_memory = 0
-        for i in range(torch.cuda.device_count()):
-            total_memory += (torch.cuda.max_memory_allocated(i) / 2**30)  # GB
-        self.max_gpu_memory_cost = max(self.max_gpu_memory_cost, total_memory)
+        if check_import('torch', raise_warning=False):
+            import torch
+            total_memory = 0
+            for i in range(torch.cuda.device_count()):
+                total_memory += (torch.cuda.max_memory_allocated(i) / 2**30)  # GB
+            self.max_gpu_memory_cost = max(self.max_gpu_memory_cost, total_memory)
+
+
+class Metrics:
+    TIME_TAKEN_FOR_TESTS = 'Time taken for tests (s)'
+    NUMBER_OF_CONCURRENCY = 'Number of concurrency'
+    TOTAL_REQUESTS = 'Total requests'
+    SUCCEED_REQUESTS = 'Succeed requests'
+    FAILED_REQUESTS = 'Failed requests'
+    OUTPUT_TOKEN_THROUGHPUT = 'Output token throughput (tok/s)'
+    TOTAL_TOKEN_THROUGHPUT = 'Total token throughput (tok/s)'
+    REQUEST_THROUGHPUT = 'Request throughput (req/s)'
+    AVERAGE_LATENCY = 'Average latency (s)'
+    AVERAGE_TIME_TO_FIRST_TOKEN = 'Average time to first token (s)'
+    AVERAGE_TIME_PER_OUTPUT_TOKEN = 'Average time per output token (s)'
+    AVERAGE_INTER_TOKEN_LATENCY = 'Average inter-token latency (s)'
+    AVERAGE_INPUT_TOKENS_PER_REQUEST = 'Average input tokens per request'
+    AVERAGE_OUTPUT_TOKENS_PER_REQUEST = 'Average output tokens per request'
 
 
 @dataclass
@@ -56,29 +77,28 @@ class BenchmarkMetrics:
     n_failed_queries: int = 0
     total_first_chunk_latency: float = 0.0
     total_latency: float = 0.0
-    n_total_chunks: int = 0
     n_total_prompt_tokens: int = 0
     n_total_completion_tokens: int = 0
-    total_chunks_time: float = 0.0
     start_time: Optional[float] = None
+    last_completed_time: Optional[float] = None
     total_time: float = 1.0
     n_total_queries: int = 0
+    n_time_per_output_token: float = 0.0
+    n_total_inter_token_latency: List[float] = field(default_factory=list)
 
     avg_first_chunk_latency: float = -1
     avg_latency: float = -1
-    n_avg_chunks: float = -1
-    avg_chunk_time: float = -1
     avg_prompt_tokens: float = -1
     avg_completion_tokens: float = -1
-    avg_token_per_seconds: float = -1
+    avg_input_token_per_seconds: float = -1
+    avg_output_token_per_seconds: float = -1
+    avg_total_token_per_seconds: float = -1
     avg_time_per_token: float = -1
+    avg_inter_token_latency: float = -1
     qps: float = -1
 
     def update_metrics(self, benchmark_data: BenchmarkData, api_plugin):
         self.n_total_queries += 1
-        if self.start_time is None:
-            self.start_time = benchmark_data.start_time
-        self.total_time = time.perf_counter() - self.start_time
 
         if benchmark_data.success:
             self.n_succeed_queries += 1
@@ -87,15 +107,30 @@ class BenchmarkMetrics:
             self.n_total_prompt_tokens += benchmark_data.prompt_tokens
             self.n_total_completion_tokens += benchmark_data.completion_tokens
 
-            benchmark_data._calculate_query_stream_metric()
             self.total_latency += benchmark_data.query_latency
             self.total_first_chunk_latency += benchmark_data.first_chunk_latency
-            self.n_total_chunks += benchmark_data.n_chunks
-            self.total_chunks_time += benchmark_data.n_chunks_time
+            self.n_time_per_output_token += benchmark_data.time_per_output_token
+            self.n_total_inter_token_latency += benchmark_data.inter_chunk_latency
         else:
             self.n_failed_queries += 1
 
         self.calculate_averages()
+        self.update_total_time(benchmark_data)
+
+    def update_total_time(self, benchmark_data: BenchmarkData):
+        # Use the earliest start_time seen so far
+        if self.start_time is None:
+            self.start_time = benchmark_data.start_time
+        else:
+            self.start_time = min(self.start_time, benchmark_data.start_time)
+        # Track the latest completion time
+        if self.last_completed_time is None:
+            self.last_completed_time = benchmark_data.completed_time
+        else:
+            self.last_completed_time = max(self.last_completed_time, benchmark_data.completed_time)
+        # Compute total_time from request lifecycle timestamps to avoid consumer overhead
+        if self.start_time is not None and self.last_completed_time is not None:
+            self.total_time = max(self.last_completed_time - self.start_time, 0.0)
 
     def calculate_averages(self):
         if self.n_succeed_queries == 0:
@@ -103,32 +138,37 @@ class BenchmarkMetrics:
         try:
             self.avg_first_chunk_latency = self.total_first_chunk_latency / self.n_succeed_queries
             self.avg_latency = self.total_latency / self.n_succeed_queries
-            self.n_avg_chunks = self.n_total_chunks / self.n_succeed_queries
-            self.avg_chunk_time = self.total_chunks_time / self.n_total_chunks
             self.avg_prompt_tokens = self.n_total_prompt_tokens / self.n_succeed_queries
             self.avg_completion_tokens = self.n_total_completion_tokens / self.n_succeed_queries
-            self.avg_token_per_seconds = self.n_total_completion_tokens / self.total_time
-            self.avg_time_per_token = self.total_time / self.n_total_completion_tokens
+            self.avg_input_token_per_seconds = self.n_total_prompt_tokens / self.total_first_chunk_latency
+            self.avg_output_token_per_seconds = self.n_total_completion_tokens / self.total_time
+            self.avg_total_token_per_seconds = (
+                self.n_total_prompt_tokens + self.n_total_completion_tokens
+            ) / self.total_time
+            self.avg_time_per_token = self.n_time_per_output_token / self.n_succeed_queries
+            self.avg_inter_token_latency = sum(self.n_total_inter_token_latency) / len(
+                self.n_total_inter_token_latency
+            ) if self.n_total_inter_token_latency else 0.0
             self.qps = self.n_succeed_queries / self.total_time
         except ZeroDivisionError as e:
             logger.exception(e)
             return
 
-    def create_message(self, default_ndigits=3):
+    def create_message(self, default_ndigits=4):
         message = {
-            'Time taken for tests (s)': round(self.total_time, default_ndigits),
-            'Number of concurrency': self.concurrency,
-            'Total requests': int(self.n_total_queries),
-            'Succeed requests': self.n_succeed_queries,
-            'Failed requests': self.n_failed_queries,
-            'Throughput(average tokens/s)': round(self.avg_token_per_seconds, default_ndigits),
-            'Average QPS': round(self.qps, default_ndigits),
-            'Average latency (s)': round(self.avg_latency, default_ndigits),
-            'Average time to first token (s)': round(self.avg_first_chunk_latency, default_ndigits),
-            'Average time per output token (s)': round(self.avg_time_per_token, 5),
-            'Average input tokens per request': round(self.avg_prompt_tokens, default_ndigits),
-            'Average output tokens per request': round(self.avg_completion_tokens, default_ndigits),
-            'Average package latency (s)': round(self.avg_chunk_time, default_ndigits),
-            'Average package per request': round(self.n_avg_chunks, default_ndigits),
+            Metrics.TIME_TAKEN_FOR_TESTS: round(self.total_time, default_ndigits),
+            Metrics.NUMBER_OF_CONCURRENCY: self.concurrency,
+            Metrics.TOTAL_REQUESTS: int(self.n_total_queries),
+            Metrics.SUCCEED_REQUESTS: self.n_succeed_queries,
+            Metrics.FAILED_REQUESTS: self.n_failed_queries,
+            Metrics.OUTPUT_TOKEN_THROUGHPUT: round(self.avg_output_token_per_seconds, default_ndigits),
+            Metrics.TOTAL_TOKEN_THROUGHPUT: round(self.avg_total_token_per_seconds, default_ndigits),
+            Metrics.REQUEST_THROUGHPUT: round(self.qps, default_ndigits),
+            Metrics.AVERAGE_LATENCY: round(self.avg_latency, default_ndigits),
+            Metrics.AVERAGE_TIME_TO_FIRST_TOKEN: round(self.avg_first_chunk_latency, default_ndigits),
+            Metrics.AVERAGE_TIME_PER_OUTPUT_TOKEN: round(self.avg_time_per_token, default_ndigits),
+            Metrics.AVERAGE_INTER_TOKEN_LATENCY: round(self.avg_inter_token_latency, default_ndigits),
+            Metrics.AVERAGE_INPUT_TOKENS_PER_REQUEST: round(self.avg_prompt_tokens, default_ndigits),
+            Metrics.AVERAGE_OUTPUT_TOKENS_PER_REQUEST: round(self.avg_completion_tokens, default_ndigits),
         }
         return message