evalscope 0.16.3__py3-none-any.whl → 0.17.1__py3-none-any.whl

evalscope/perf/plugin/datasets/line_by_line.py

@@ -20,6 +20,7 @@ class LineByLineDatasetPlugin(DatasetPluginBase):
             if len(prompt) > self.query_parameters.min_prompt_length and len(
                     prompt) < self.query_parameters.max_prompt_length:
                 if self.query_parameters.apply_chat_template:
-                    yield [{'role': 'user', 'content': prompt}]
+                    message = self.create_message(prompt)
+                    yield [message]
                 else:
                     yield prompt

evalscope/perf/plugin/datasets/longalpaca.py

@@ -25,6 +25,7 @@ class LongAlpacaDatasetPlugin(DatasetPluginBase):
             if len(prompt) > self.query_parameters.min_prompt_length and len(
                     prompt) < self.query_parameters.max_prompt_length:
                 if self.query_parameters.apply_chat_template:
-                    yield [{'role': 'user', 'content': prompt}]
+                    message = self.create_message(prompt)
+                    yield [message]
                 else:
                     yield prompt

evalscope/perf/plugin/datasets/openqa.py

@@ -30,6 +30,7 @@ class OpenqaDatasetPlugin(DatasetPluginBase):
             if (len(prompt) > self.query_parameters.min_prompt_length
                     and len(prompt) < self.query_parameters.max_prompt_length):
                 if self.query_parameters.apply_chat_template:
-                    yield [{'role': 'user', 'content': prompt}]
+                    message = self.create_message(prompt)
+                    yield [message]
                 else:
                     yield prompt

evalscope/perf/plugin/datasets/random_dataset.py

@@ -37,12 +37,23 @@ class RandomDatasetPlugin(DatasetPluginBase):
         input_lens = np.random.randint(min_prompt_length, max_prompt_length, size=self.number)
         offsets = np.random.randint(0, self.tokenizer.vocab_size, size=self.number)
 
+        vocab_size = self.tokenizer.vocab_size
+
         for i in range(self.number):
-            prompt_ids = ((offsets[i] + i + np.arange(input_lens[i])) % self.tokenizer.vocab_size).tolist()
-            prompt = self.tokenizer.decode(self.prefix_ids + prompt_ids)
+            inner_seq = ((offsets[i] + i + np.arange(input_lens[i])) % vocab_size).tolist()
+            token_sequence = self.prefix_ids + inner_seq
+            prompt = self.tokenizer.decode(token_sequence)
+
+            # After decoding the prompt we have to encode and decode it again.
+            # This is done because in some cases N consecutive tokens
+            # give a string tokenized into != N number of tokens.
+            total_input_len = self.prefix_length + int(input_lens[i])
+            re_encoded_sequence = self.tokenizer.encode(prompt, add_special_tokens=False)[:total_input_len]
+            prompt = self.tokenizer.decode(re_encoded_sequence)
 
             if self.query_parameters.apply_chat_template:
-                yield [{'role': 'user', 'content': prompt}]
+                message = self.create_message(prompt)
+                yield [message]
             else:
                 yield prompt
 
@@ -53,6 +64,6 @@ class RandomDatasetPlugin(DatasetPluginBase):
         return input_ids
 
     def get_template_len(self):
-        empty_message = [{'role': 'user', 'content': ''}]
+        empty_message = [self.create_message(text='')]
         template = self.tokenizer.apply_chat_template(empty_message, tokenize=True, add_generation_prompt=True)
         return len(template)

evalscope/perf/plugin/datasets/random_vl_dataset.py

@@ -0,0 +1,80 @@
+import random
+from PIL import Image, ImageDraw
+from typing import Dict, Iterator, List
+
+from evalscope.perf.arguments import Arguments
+from evalscope.perf.plugin.datasets.random_dataset import RandomDatasetPlugin
+from evalscope.perf.plugin.registry import register_dataset
+from evalscope.utils.io_utils import PIL_to_base64
+
+
+@register_dataset('random_vl')
+class RandomVLDatasetPlugin(RandomDatasetPlugin):
+    """Random Vision-Language Dataset Plugin for multimodal model stress testing."""
+
+    def __init__(self, query_parameters: Arguments):
+        super().__init__(query_parameters)
+
+        # Vision-language specific parameters
+        self.image_width = query_parameters.image_width
+        self.image_height = query_parameters.image_height
+        self.image_format = query_parameters.image_format
+        self.image_num = query_parameters.image_num
+
+        assert self.image_num > 0, 'image_num must be greater than 0.'
+
+    def build_messages(self) -> Iterator[List[Dict]]:
+        # Reuse parent's message generation logic
+        for messages in super().build_messages():
+            prompt = messages[0]['content'] if isinstance(messages[0], dict) else messages[0]
+
+            # Generate random images based on image_num
+            images_b64 = []
+            for _ in range(self.image_num):
+                images_b64.append(f'data:image/png;base64,{self._generate_random_image_b64()}')
+
+            message = self.create_message(text=prompt, image_urls=images_b64)
+            yield [message]
+
+    def _generate_random_image_b64(self) -> str:
+        """Generate a random image and return as base64 string."""
+        # Create a random colored image
+        color = (random.randint(0, 255), random.randint(0, 255), random.randint(0, 255))
+        image = Image.new(self.image_format, (self.image_width, self.image_height), color)
+
+        # Add some random shapes for variety
+        draw = ImageDraw.Draw(image)
+        for _ in range(random.randint(1, 5)):
+            shape_type = random.choice(['rectangle', 'ellipse', 'line'])
+
+            # Generate two random points
+            x1 = random.randint(0, self.image_width - 1)
+            y1 = random.randint(0, self.image_height - 1)
+            x2 = random.randint(0, self.image_width - 1)
+            y2 = random.randint(0, self.image_height - 1)
+
+            # Ensure proper coordinate ordering (x1 <= x2, y1 <= y2)
+            if x1 > x2:
+                x1, x2 = x2, x1
+            if y1 > y2:
+                y1, y2 = y2, y1
+
+            # Ensure we have at least a 1-pixel difference
+            if x1 == x2:
+                x2 = min(x1 + 1, self.image_width - 1)
+            if y1 == y2:
+                y2 = min(y1 + 1, self.image_height - 1)
+
+            coords = [x1, y1, x2, y2]
+
+            shape_color = (random.randint(0, 255), random.randint(0, 255), random.randint(0, 255))
+
+            if shape_type == 'rectangle':
+                draw.rectangle(coords, fill=shape_color)
+            elif shape_type == 'ellipse':
+                draw.ellipse(coords, fill=shape_color)
+            else:
+                draw.line(coords, fill=shape_color, width=random.randint(1, 5))
+
+        # Convert to base64
+        return PIL_to_base64(image, format='PNG')

evalscope/perf/plugin/registry.py

@@ -1,23 +1,25 @@
-from typing import Any, List, Type, Union
+from typing import TYPE_CHECKING, Any, List, Type, Union
 
+if TYPE_CHECKING:
+    from .api import ApiPluginBase
+    from .datasets import DatasetPluginBase
 
-class PluginRegistry:
-
-    def __init__(self):
-        self._registry = {}
 
-    def register(self, name, cls):
-        self._registry[name] = cls
-        return cls
+class PluginRegistry:
+    _registry = {}
 
-    def get_class(self, name):
-        return self._registry[name]
+    @classmethod
+    def register(cls, name, plugin_cls):
+        cls._registry[name] = plugin_cls
+        return plugin_cls
 
-    def all_classes(self):
-        return list(self._registry.keys())
+    @classmethod
+    def get_class(cls, name):
+        return cls._registry[name]
 
-    def __call__(self, name: str) -> Any:
-        return self.get_class(name)
+    @classmethod
+    def all_classes(cls):
+        return list(cls._registry.keys())
 
 
 def register_dataset(name: Union[str, List[str]]):
@@ -50,5 +52,23 @@ def register_api(name: Union[str, List[str]]):
     return class_decorator
 
 
-DatasetRegistry = PluginRegistry()
-ApiRegistry = PluginRegistry()
+class DatasetRegistry(PluginRegistry):
+    """Registry for dataset plugins."""
+    _registry = {}
+
+    @classmethod
+    def get_class(cls, name: str) -> Type['DatasetPluginBase']:
+        if name not in cls._registry:
+            raise ValueError(f"Dataset plugin '{name}' is not registered.")
+        return cls._registry[name]
+
+
+class ApiRegistry(PluginRegistry):
+    """Registry for API plugins."""
+    _registry = {}
+
+    @classmethod
+    def get_class(cls, name: str) -> Type['ApiPluginBase']:
+        if name not in cls._registry:
+            raise ValueError(f"API plugin '{name}' is not registered.")
+        return cls._registry[name]

evalscope/perf/utils/analysis_result.py

@@ -3,27 +3,28 @@ import json
 import pickle
 import sqlite3
 
-result_db_path = './outputs/qwen2.5_benchmark_20241111_160543.db'
-con = sqlite3.connect(result_db_path)
-query_sql = "SELECT request, response_messages, prompt_tokens, completion_tokens \
-                FROM result WHERE success='1'"
+db_path = 'your db path'
+conn = sqlite3.connect(db_path)
+cursor = conn.cursor()
 
-# how to save base64.b64encode(pickle.dumps(benchmark_data["request"])).decode("ascii"),
-with con:
-    rows = con.execute(query_sql).fetchall()
-    if len(rows) > 0:
-        for row in rows:
-            request = row[0]
-            responses = row[1]
-            request = base64.b64decode(request)
-            request = pickle.loads(request)
-            responses = base64.b64decode(responses)
-            responses = pickle.loads(responses)
-            response_content = ''
-            for response in responses:
-                response = json.loads(response)
-                if not response['choices']:
-                    continue
-                response_content += response['choices'][0]['delta']['content']
-            print('prompt: %s, tokens: %s, completion: %s, tokens: %s' %
-                  (request['messages'][0]['content'], row[2], response_content, row[3]))
+# 获取列名
+cursor.execute('PRAGMA table_info(result)')
+columns = [info[1] for info in cursor.fetchall()]
+print('列名：', columns)
+
+cursor.execute('SELECT * FROM result WHERE success=1 AND first_chunk_latency > 1')
+rows = cursor.fetchall()
+print(f'len(rows): {len(rows)}')
+
+for row in rows:
+    row_dict = dict(zip(columns, row))
+    # 解码request
+    row_dict['request'] = pickle.loads(base64.b64decode(row_dict['request']))
+    # 解码response_messages
+    row_dict['response_messages'] = pickle.loads(base64.b64decode(row_dict['response_messages']))
+    # print(row_dict)
+    print(
+        f"request_id: {json.loads(row_dict['response_messages'][0])['id']}, first_chunk_latency: {row_dict['first_chunk_latency']}"  # noqa: E501
+    )
+    # 如果只想看一个可以break
+    # break

evalscope/perf/utils/benchmark_util.py

@@ -20,25 +20,24 @@ class BenchmarkData:
     # late init
     query_latency: float = 0.0
     first_chunk_latency: float = 0.0
-    n_chunks: int = 0
-    n_chunks_time: float = 0.0
     max_gpu_memory_cost = 0
     time_per_output_token: float = 0.0
+    inter_chunk_latency: List[float] = field(default_factory=list)
 
     prompt_tokens = None
     completion_tokens = None
 
-    def _calculate_query_stream_metric(self) -> Tuple[float, int, float]:
+    def _calculate_query_stream_metric(self) -> None:
         self.query_latency = self.completed_time - self.start_time
+        # only for stream responses
         if len(self.chunk_times) > 1:
             self.first_chunk_latency = self.chunk_times[0] - self.start_time
-            self.n_chunks = len(self.chunk_times) - 2  # remove last and first chunk
-            self.n_chunks_time = self.chunk_times[-2] - self.chunk_times[0]
+            # remove the first chunk time from the total latency
+            self.time_per_output_token = (self.query_latency - self.first_chunk_latency) / (
+                self.completion_tokens - 1) if self.completion_tokens > 1 else 0.0
+            self.inter_chunk_latency = [t2 - t1 for t1, t2 in zip(self.chunk_times[:-1], self.chunk_times[1:])]
         else:
             self.first_chunk_latency = self.query_latency
-            self.n_chunks = 1
-            self.n_chunks_time = self.query_latency
-        self.time_per_output_token = self.n_chunks_time / self.n_chunks
 
     def _calculate_tokens(self, api_plugin):
         self.prompt_tokens, self.completion_tokens = \
@@ -63,10 +62,9 @@ class Metrics:
     AVERAGE_LATENCY = 'Average latency (s)'
     AVERAGE_TIME_TO_FIRST_TOKEN = 'Average time to first token (s)'
     AVERAGE_TIME_PER_OUTPUT_TOKEN = 'Average time per output token (s)'
+    AVERAGE_INTER_TOKEN_LATENCY = 'Average inter-token latency (s)'
     AVERAGE_INPUT_TOKENS_PER_REQUEST = 'Average input tokens per request'
     AVERAGE_OUTPUT_TOKENS_PER_REQUEST = 'Average output tokens per request'
-    AVERAGE_PACKAGE_LATENCY = 'Average package latency (s)'
-    AVERAGE_PACKAGE_PER_REQUEST = 'Average package per request'
 
 
 @dataclass
@@ -76,25 +74,23 @@ class BenchmarkMetrics:
     n_failed_queries: int = 0
     total_first_chunk_latency: float = 0.0
     total_latency: float = 0.0
-    n_total_chunks: int = 0
     n_total_prompt_tokens: int = 0
     n_total_completion_tokens: int = 0
-    total_chunks_time: float = 0.0
     start_time: Optional[float] = None
     total_time: float = 1.0
     n_total_queries: int = 0
     n_time_per_output_token: float = 0.0
+    n_total_inter_token_latency: List[float] = field(default_factory=list)
 
     avg_first_chunk_latency: float = -1
     avg_latency: float = -1
-    n_avg_chunks: float = -1
-    avg_chunk_time: float = -1
     avg_prompt_tokens: float = -1
     avg_completion_tokens: float = -1
     avg_input_token_per_seconds: float = -1
     avg_output_token_per_seconds: float = -1
     avg_total_token_per_seconds: float = -1
     avg_time_per_token: float = -1
+    avg_inter_token_latency: float = -1
     qps: float = -1
 
     def update_metrics(self, benchmark_data: BenchmarkData, api_plugin):
@@ -113,9 +109,8 @@ class BenchmarkMetrics:
             benchmark_data._calculate_query_stream_metric()
             self.total_latency += benchmark_data.query_latency
             self.total_first_chunk_latency += benchmark_data.first_chunk_latency
-            self.n_total_chunks += benchmark_data.n_chunks
-            self.total_chunks_time += benchmark_data.n_chunks_time
             self.n_time_per_output_token += benchmark_data.time_per_output_token
+            self.n_total_inter_token_latency += benchmark_data.inter_chunk_latency
         else:
             self.n_failed_queries += 1
 
@@ -127,8 +122,6 @@ class BenchmarkMetrics:
         try:
             self.avg_first_chunk_latency = self.total_first_chunk_latency / self.n_succeed_queries
             self.avg_latency = self.total_latency / self.n_succeed_queries
-            self.n_avg_chunks = self.n_total_chunks / self.n_succeed_queries
-            self.avg_chunk_time = self.total_chunks_time / self.n_total_chunks
             self.avg_prompt_tokens = self.n_total_prompt_tokens / self.n_succeed_queries
             self.avg_completion_tokens = self.n_total_completion_tokens / self.n_succeed_queries
             self.avg_input_token_per_seconds = self.n_total_prompt_tokens / self.total_first_chunk_latency
@@ -136,6 +129,8 @@ class BenchmarkMetrics:
             self.avg_total_token_per_seconds = (self.n_total_prompt_tokens
                                                 + self.n_total_completion_tokens) / self.total_time
             self.avg_time_per_token = self.n_time_per_output_token / self.n_succeed_queries
+            self.avg_inter_token_latency = sum(self.n_total_inter_token_latency) / len(
+                self.n_total_inter_token_latency) if self.n_total_inter_token_latency else 0.0
             self.qps = self.n_succeed_queries / self.total_time
         except ZeroDivisionError as e:
             logger.exception(e)
@@ -154,9 +149,8 @@ class BenchmarkMetrics:
             Metrics.AVERAGE_LATENCY: round(self.avg_latency, default_ndigits),
             Metrics.AVERAGE_TIME_TO_FIRST_TOKEN: round(self.avg_first_chunk_latency, default_ndigits),
             Metrics.AVERAGE_TIME_PER_OUTPUT_TOKEN: round(self.avg_time_per_token, default_ndigits),
+            Metrics.AVERAGE_INTER_TOKEN_LATENCY: round(self.avg_inter_token_latency, default_ndigits),
             Metrics.AVERAGE_INPUT_TOKENS_PER_REQUEST: round(self.avg_prompt_tokens, default_ndigits),
             Metrics.AVERAGE_OUTPUT_TOKENS_PER_REQUEST: round(self.avg_completion_tokens, default_ndigits),
-            Metrics.AVERAGE_PACKAGE_LATENCY: round(self.avg_chunk_time, default_ndigits),
-            Metrics.AVERAGE_PACKAGE_PER_REQUEST: round(self.n_avg_chunks, default_ndigits),
         }
         return message

evalscope/perf/utils/db_util.py

@@ -16,6 +16,28 @@ from evalscope.utils.logger import get_logger
 logger = get_logger()
 
 
+class DatabaseColumns:
+    REQUEST = 'request'
+    START_TIME = 'start_time'
+    CHUNK_TIMES = 'chunk_times'
+    SUCCESS = 'success'
+    RESPONSE_MESSAGES = 'response_messages'
+    COMPLETED_TIME = 'completed_time'
+    LATENCY = 'latency'
+    FIRST_CHUNK_LATENCY = 'first_chunk_latency'
+    PROMPT_TOKENS = 'prompt_tokens'
+    COMPLETION_TOKENS = 'completion_tokens'
+    MAX_GPU_MEMORY_COST = 'max_gpu_memory_cost'
+    TIME_PER_OUTPUT_TOKEN = 'time_per_output_token'
+
+
+def load_prompt(prompt_path_or_text):
+    if prompt_path_or_text.startswith('@'):
+        with open(prompt_path_or_text[1:], 'r', encoding='utf-8') as file:
+            return file.read()
+    return prompt_path_or_text
+
+
 def encode_data(data) -> str:
     """Encodes data using base64 and pickle."""
     return base64.b64encode(pickle.dumps(data)).decode('utf-8')
@@ -34,20 +56,20 @@ def transpose_results(data):
 
 
 def create_result_table(cursor):
-    cursor.execute('''CREATE TABLE IF NOT EXISTS result(
-                      request TEXT,
-                      start_time REAL,
-                      chunk_times TEXT,
-                      success INTEGER,
-                      response_messages TEXT,
-                      completed_time REAL,
-                      latency REAL,
-                      first_chunk_latency REAL,
-                      n_chunks INTEGER,
-                      chunk_time REAL,
-                      prompt_tokens INTEGER,
-                      completion_tokens INTEGER,
-                      max_gpu_memory_cost REAL)''')
+    cursor.execute(f'''CREATE TABLE IF NOT EXISTS result(
+                      {DatabaseColumns.REQUEST} TEXT,
+                      {DatabaseColumns.START_TIME} REAL,
+                      {DatabaseColumns.CHUNK_TIMES} TEXT,
+                      {DatabaseColumns.SUCCESS} INTEGER,
+                      {DatabaseColumns.RESPONSE_MESSAGES} TEXT,
+                      {DatabaseColumns.COMPLETED_TIME} REAL,
+                      {DatabaseColumns.LATENCY} REAL,
+                      {DatabaseColumns.FIRST_CHUNK_LATENCY} REAL,
+                      {DatabaseColumns.PROMPT_TOKENS} INTEGER,
+                      {DatabaseColumns.COMPLETION_TOKENS} INTEGER,
+                      {DatabaseColumns.MAX_GPU_MEMORY_COST} REAL,
+                      {DatabaseColumns.TIME_PER_OUTPUT_TOKEN} REAL
+                   )''')
 
 
 def insert_benchmark_data(cursor: sqlite3.Cursor, benchmark_data: BenchmarkData):
@@ -67,24 +89,21 @@ def insert_benchmark_data(cursor: sqlite3.Cursor, benchmark_data: BenchmarkData)
 
     if benchmark_data.success:
         # Add additional columns for success case
-        additional_columns = (
-            benchmark_data.query_latency,
-            benchmark_data.first_chunk_latency,
-            benchmark_data.n_chunks,
-            benchmark_data.n_chunks_time,
-            benchmark_data.prompt_tokens,
-            benchmark_data.completion_tokens,
-            benchmark_data.max_gpu_memory_cost,
-        )
-        query = """INSERT INTO result(
-                      request, start_time, chunk_times, success, response_messages,
-                      completed_time, latency, first_chunk_latency,
-                      n_chunks, chunk_time, prompt_tokens, completion_tokens, max_gpu_memory_cost
-                   ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"""
+        additional_columns = (benchmark_data.query_latency, benchmark_data.first_chunk_latency,
+                              benchmark_data.prompt_tokens, benchmark_data.completion_tokens,
+                              benchmark_data.max_gpu_memory_cost, benchmark_data.time_per_output_token)
+        query = f"""INSERT INTO result(
+                      {DatabaseColumns.REQUEST}, {DatabaseColumns.START_TIME}, {DatabaseColumns.CHUNK_TIMES},
+                      {DatabaseColumns.SUCCESS}, {DatabaseColumns.RESPONSE_MESSAGES}, {DatabaseColumns.COMPLETED_TIME},
+                      {DatabaseColumns.LATENCY}, {DatabaseColumns.FIRST_CHUNK_LATENCY}, {DatabaseColumns.PROMPT_TOKENS},
+                      {DatabaseColumns.COMPLETION_TOKENS}, {DatabaseColumns.MAX_GPU_MEMORY_COST},
+                      {DatabaseColumns.TIME_PER_OUTPUT_TOKEN}
+                   ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"""
         cursor.execute(query, common_columns + additional_columns)
     else:
-        query = """INSERT INTO result(
-                      request, start_time, chunk_times, success, response_messages, completed_time
+        query = f"""INSERT INTO result(
+                      {DatabaseColumns.REQUEST}, {DatabaseColumns.START_TIME}, {DatabaseColumns.CHUNK_TIMES},
+                      {DatabaseColumns.SUCCESS}, {DatabaseColumns.RESPONSE_MESSAGES}, {DatabaseColumns.COMPLETED_TIME}
                    ) VALUES (?, ?, ?, ?, ?, ?)"""
         cursor.execute(query, common_columns)
 
@@ -160,44 +179,43 @@ def get_percentile_results(result_db_path: str) -> Dict[str, List[float]]:
             logger.error(f'Error parsing chunk times: {e}')
             return []
 
-    query_sql = ('SELECT start_time, chunk_times, success, completed_time, latency, first_chunk_latency, '
-                 'n_chunks, chunk_time, prompt_tokens, completion_tokens '
-                 'FROM result WHERE success=1')
+    query_sql = f'''SELECT {DatabaseColumns.START_TIME}, {DatabaseColumns.CHUNK_TIMES}, {DatabaseColumns.SUCCESS},
+                    {DatabaseColumns.COMPLETED_TIME}, {DatabaseColumns.LATENCY}, {DatabaseColumns.FIRST_CHUNK_LATENCY},
+                    {DatabaseColumns.PROMPT_TOKENS},
+                    {DatabaseColumns.COMPLETION_TOKENS}, {DatabaseColumns.TIME_PER_OUTPUT_TOKEN}
+                    FROM result WHERE {DatabaseColumns.SUCCESS}=1'''
 
     percentiles = [10, 25, 50, 66, 75, 80, 90, 95, 98, 99]
 
     with sqlite3.connect(result_db_path) as con:
-        rows = con.execute(query_sql).fetchall()
+        cursor = con.cursor()
+        cursor.execute(query_sql)
+        columns = [description[0] for description in cursor.description]
+        rows = cursor.fetchall()
 
-    # Define index variables for columns
-    CHUNK_TIMES_INDEX = 1
-    LATENCY_INDEX = 4
-    FIRST_CHUNK_LATENCY_INDEX = 5
-    CHUNK_TIME_INDEX = 7
-    PROMPT_TOKENS_INDEX = 8
-    COMPLETION_TOKENS_INDEX = 9
+    # Create column index mapping
+    col_indices = {col: idx for idx, col in enumerate(columns)}
 
     # Prepare data for each metric
     inter_token_latencies_all = []
     for row in rows:
-        inter_token_latencies_all.extend(inter_token_latencies(row[CHUNK_TIMES_INDEX]))
+        inter_token_latencies_all.extend(inter_token_latencies(row[col_indices[DatabaseColumns.CHUNK_TIMES]]))
 
     metrics = {
-        PercentileMetrics.TTFT: [row[FIRST_CHUNK_LATENCY_INDEX] for row in rows],
+        PercentileMetrics.TTFT: [row[col_indices[DatabaseColumns.FIRST_CHUNK_LATENCY]] for row in rows],
         PercentileMetrics.ITL:
         inter_token_latencies_all,
-        PercentileMetrics.TPOT:
-        [(row[CHUNK_TIME_INDEX] / row[COMPLETION_TOKENS_INDEX]) if row[COMPLETION_TOKENS_INDEX] > 0 else float('nan')
-         for row in rows],
-        PercentileMetrics.LATENCY: [row[LATENCY_INDEX] for row in rows],
-        PercentileMetrics.INPUT_TOKENS: [row[PROMPT_TOKENS_INDEX] for row in rows],
-        PercentileMetrics.OUTPUT_TOKENS: [row[COMPLETION_TOKENS_INDEX] for row in rows],
+        PercentileMetrics.TPOT: [row[col_indices[DatabaseColumns.TIME_PER_OUTPUT_TOKEN]] for row in rows],
+        PercentileMetrics.LATENCY: [row[col_indices[DatabaseColumns.LATENCY]] for row in rows],
+        PercentileMetrics.INPUT_TOKENS: [row[col_indices[DatabaseColumns.PROMPT_TOKENS]] for row in rows],
+        PercentileMetrics.OUTPUT_TOKENS: [row[col_indices[DatabaseColumns.COMPLETION_TOKENS]] for row in rows],
         PercentileMetrics.OUTPUT_THROUGHPUT:
-        [(row[COMPLETION_TOKENS_INDEX] / row[LATENCY_INDEX]) if row[LATENCY_INDEX] > 0 else float('nan')
-         for row in rows],
-        PercentileMetrics.TOTAL_THROUGHPUT: [((row[PROMPT_TOKENS_INDEX] + row[COMPLETION_TOKENS_INDEX])
-                                              / row[LATENCY_INDEX]) if row[LATENCY_INDEX] > 0 else float('nan')
-                                             for row in rows]
+        [(row[col_indices[DatabaseColumns.COMPLETION_TOKENS]] / row[col_indices[DatabaseColumns.LATENCY]])
+         if row[col_indices[DatabaseColumns.LATENCY]] > 0 else float('nan') for row in rows],
+        PercentileMetrics.TOTAL_THROUGHPUT:
+        [((row[col_indices[DatabaseColumns.PROMPT_TOKENS]] + row[col_indices[DatabaseColumns.COMPLETION_TOKENS]])
+          / row[col_indices[DatabaseColumns.LATENCY]])
+         if row[col_indices[DatabaseColumns.LATENCY]] > 0 else float('nan') for row in rows]
     }
 
     # Calculate percentiles for each metric
@@ -237,18 +255,18 @@ def summary_result(args: Arguments, metrics: BenchmarkMetrics, result_db_path: s
 
 
 def speed_benchmark_result(result_db_path: str):
-    query_sql = """
+    query_sql = f"""
         SELECT
-            prompt_tokens,
-            ROUND(AVG(completion_tokens / latency), 2) AS avg_completion_token_per_second,
-            ROUND(AVG(max_gpu_memory_cost), 2)
+            {DatabaseColumns.PROMPT_TOKENS},
+            ROUND(AVG({DatabaseColumns.COMPLETION_TOKENS} / {DatabaseColumns.LATENCY}), 2) AS avg_completion_token_per_second,
+            ROUND(AVG({DatabaseColumns.MAX_GPU_MEMORY_COST}), 2)
         FROM
             result
         WHERE
-            success = 1 AND latency > 0
+            {DatabaseColumns.SUCCESS} = 1 AND {DatabaseColumns.LATENCY} > 0
         GROUP BY
-            prompt_tokens
-    """
+            {DatabaseColumns.PROMPT_TOKENS}
+    """  # noqa: E501
 
     with sqlite3.connect(result_db_path) as con:
         cursor = con.cursor()

evalscope/report/__init__.py

@@ -4,7 +4,7 @@ from typing import TYPE_CHECKING
 from evalscope.utils.import_utils import _LazyModule
 
 if TYPE_CHECKING:
-    from .combinator import gen_report_table, gen_table, get_data_frame, get_report_list
+    from .combinator import gen_table, get_data_frame, get_report_list
     from .generator import ReportGenerator
     from .utils import Category, Report, ReportKey, Subset
 

evalscope/report/utils.py

@@ -3,14 +3,45 @@ import os
 import pandas as pd
 from collections import defaultdict
 from dataclasses import asdict, dataclass, field
-from typing import Any, Dict, List
+from typing import Any, Dict, List, Union
 
 from evalscope.metrics import macro_mean, micro_mean
-from evalscope.utils import normalize_score
-from evalscope.utils.logger import get_logger
+from evalscope.utils import get_logger
 
 logger = get_logger()
 
+ANALYSIS_PROMPT = """根据给出的json格式的模型评测结果，输出分析报告，要求如下：
+1. 报告分为 总体表现、关键指标分析、改进建议、结论 四部分
+2. 若模型有多种指标，将其分为低分、中分、高分三个部分，并列出markdown表格
+3. 只列出报告本身，不要有其他多余内容
+4. 输出报告语言为{language}
+
+```json
+{report_str}
+```
+"""
+
+
+def normalize_score(score: Union[float, dict], keep_num: int = 4) -> Union[float, dict]:
+    """
+    Normalize score.
+
+    Args:
+        score: input score, could be float or dict. e.g. 0.12345678 or {'acc': 0.12345678, 'f1': 0.12345678}
+        keep_num: number of digits to keep.
+
+    Returns:
+        Union[float, dict]: normalized score. e.g. 0.1234 or {'acc': 0.1234, 'f1': 0.1234}
+    """
+    if isinstance(score, float):
+        score = round(score, keep_num)
+    elif isinstance(score, dict):
+        score = {k: round(v, keep_num) for k, v in score.items()}
+    else:
+        logger.warning(f'Unknown score type: {type(score)}')
+
+    return score
+
 
 @dataclass
 class Subset:
@@ -74,18 +105,6 @@ class ReportKey:
     score = 'Score'
 
 
-ANALYSIS_PROMPT = """根据给出的json格式的模型评测结果，输出分析报告，要求如下：
-1. 报告分为 总体表现、关键指标分析、改进建议、结论 四部分
-2. 若模型有多种指标，将其分为低分、中分、高分三个部分，并列出markdown表格
-3. 只列出报告本身，不要有其他多余内容
-4. 输出报告语言为{language}
-
-```json
-{report_str}
-```
-"""
-
-
 @dataclass
 class Report:
     name: str = 'default_report'