evalscope 0.11.0__py3-none-any.whl → 0.12.1__py3-none-any.whl

evalscope/metrics/named_metrics.py

@@ -35,6 +35,7 @@ metric_registry = MetricRegistry()
 metric_registry.register(Metric(name='AverageAccuracy', object=mean))
 metric_registry.register(Metric(name='WeightedAverageAccuracy', object=weighted_mean))
 metric_registry.register(Metric(name='AverageBLEU', object=mean))
+metric_registry.register(Metric(name='AverageRouge', object=mean))
 metric_registry.register(Metric(name='WeightedAverageBLEU', object=weighted_mean))
 metric_registry.register(Metric(name='AveragePass@1', object=mean))
 for k in range(1, 17):

evalscope/models/__init__.py

@@ -7,10 +7,11 @@ from evalscope.models.custom import CustomModel
 from evalscope.models.custom_adapter import CustomModelAdapter
 from evalscope.models.local_model import LocalModel, get_local_model
 from evalscope.models.model import BaseModel, ChatBaseModel, OpenAIModel
+from evalscope.models.register import get_model_adapter
 from evalscope.models.server_adapter import ServerModelAdapter
 
 __all__ = [
     'CustomModel', 'BaseModel', 'ChatBaseModel', 'OpenAIModel', 'BaseModelAdapter', 'ChatGenerationModelAdapter',
     'MultiChoiceModelAdapter', 'ContinuationLogitsModelAdapter', 'CustomModelAdapter', 'ServerModelAdapter',
-    'LocalModel', 'get_local_model', 'initialize_model_adapter'
+    'LocalModel', 'get_local_model', 'initialize_model_adapter', 'get_model_adapter'
 ]

evalscope/models/base_adapter.py

@@ -1,15 +1,21 @@
 import torch
 from abc import ABC, abstractmethod
-from typing import TYPE_CHECKING, Any, Optional, Union
+from typing import TYPE_CHECKING, Any, List, Optional, Union
 
-from evalscope.constants import EvalType
+from evalscope.constants import EvalType, OutputType
 from evalscope.models.custom import CustomModel
 from evalscope.models.local_model import LocalModel
+from evalscope.models.register import get_model_adapter, register_model_adapter
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
 
 if TYPE_CHECKING:
+    from evalscope.benchmarks import BenchmarkMeta
     from evalscope.config import TaskConfig
 
 
+@register_model_adapter('base')
 class BaseModelAdapter(ABC):
 
     def __init__(self, model: Optional[Union[LocalModel, CustomModel]], **kwargs):
@@ -33,7 +39,7 @@ class BaseModelAdapter(ABC):
         raise NotImplementedError
 
 
-def initialize_model_adapter(task_cfg: 'TaskConfig', model_adapter_cls: 'BaseModelAdapter', base_model: 'LocalModel'):
+def initialize_model_adapter(task_cfg: 'TaskConfig', benchmark: 'BenchmarkMeta', base_model: 'LocalModel'):
     """Initialize the model adapter based on the task configuration."""
     if task_cfg.dry_run:
         from evalscope.models.model import DummyChatModel
@@ -43,10 +49,30 @@ def initialize_model_adapter(task_cfg: 'TaskConfig', model_adapter_cls: 'BaseMod
             raise ValueError(f'Expected evalscope.models.custom.CustomModel, but got {type(task_cfg.model)}.')
         from evalscope.models import CustomModelAdapter
         return CustomModelAdapter(custom_model=task_cfg.model)
-    elif task_cfg.eval_type == EvalType.SERVICE:
+    elif task_cfg.eval_type == EvalType.SERVICE or task_cfg.api_url is not None:
         from evalscope.models import ServerModelAdapter
+
+        if benchmark.model_adapter in [OutputType.CONTINUOUS, OutputType.MULTIPLE_CHOICE]:
+            logger.warning('Output type is set to logits. This is not supported for service evaluation. '
+                           'Setting output type to generation by default.')
+            benchmark.model_adapter = OutputType.GENERATION
+
         return ServerModelAdapter(
-            api_url=task_cfg.api_url, model_id=task_cfg.model, api_key=task_cfg.api_key, seed=task_cfg.seed)
+            api_url=task_cfg.api_url,
+            model_id=task_cfg.model,
+            api_key=task_cfg.api_key,
+            seed=task_cfg.seed,
+            timeout=task_cfg.timeout,
+            stream=task_cfg.stream,
+        )
     else:
-        return model_adapter_cls(
+        # for local model, we need to determine the model adapter class based on the output type
+        model_adapter_cls = benchmark.model_adapter
+        if model_adapter_cls not in benchmark.output_types:
+            logger.warning(f'Output type {model_adapter_cls} is not supported for benchmark {benchmark.name}. '
+                           f'Using {benchmark.output_types[0]} instead.')
+            model_adapter_cls = benchmark.output_types[0]
+
+        model_adapter = get_model_adapter(model_adapter_cls)
+        return model_adapter(
             model=base_model, generation_config=task_cfg.generation_config, chat_template=task_cfg.chat_template)

evalscope/models/chat_adapter.py

@@ -3,8 +3,10 @@ import time
 import torch
 from typing import List, Union
 
+from evalscope.constants import OutputType
 from evalscope.models.base_adapter import BaseModelAdapter
 from evalscope.models.local_model import LocalModel
+from evalscope.models.register import register_model_adapter
 from evalscope.utils.chat_service import ChatCompletionResponse, ChatCompletionResponseChoice, ChatMessage
 from evalscope.utils.logger import get_logger
 from evalscope.utils.model_utils import fix_do_sample_warning
@@ -12,6 +14,7 @@ from evalscope.utils.model_utils import fix_do_sample_warning
 logger = get_logger()
 
 
+@register_model_adapter(OutputType.GENERATION)
 class ChatGenerationModelAdapter(BaseModelAdapter):
     """
     Chat generation model adapter.
@@ -102,7 +105,7 @@ class ChatGenerationModelAdapter(BaseModelAdapter):
         # Get input ids
         inputs = self.tokenizer(
             formatted_prompts, return_tensors='pt', padding=True, truncation=True,
-            padding_side='left').to(self.device)  # padding_side='left' is important for chat model
+            padding_side='left').to(self.model.device)  # padding_side='left' is important for chat model
         input_ids = inputs['input_ids']
 
         # Run inference

evalscope/models/choice_adapter.py

@@ -3,11 +3,14 @@ import time
 import torch
 from typing import List
 
+from evalscope.constants import OutputType
 from evalscope.models.base_adapter import BaseModelAdapter
 from evalscope.models.local_model import LocalModel
+from evalscope.models.register import register_model_adapter
 from evalscope.utils.chat_service import ChatCompletionResponse, ChatCompletionResponseChoice, ChatMessage
 
 
+@register_model_adapter(OutputType.MULTIPLE_CHOICE)
 class MultiChoiceModelAdapter(BaseModelAdapter):
     """ The multi-choice model adapter. """
 
@@ -110,6 +113,7 @@ class MultiChoiceModelAdapter(BaseModelAdapter):
         return log_probs, {'tokens': tokens}
 
 
+@register_model_adapter(OutputType.CONTINUOUS)
 class ContinuationLogitsModelAdapter(MultiChoiceModelAdapter):
     """
     Continuation-logits model adapter.

evalscope/models/custom_adapter.py

@@ -2,8 +2,10 @@ from typing import Any, Dict, List, Union
 
 from evalscope.models.base_adapter import BaseModelAdapter
 from evalscope.models.custom import CustomModel
+from evalscope.models.register import register_model_adapter
 
 
+@register_model_adapter('custom')
 class CustomModelAdapter(BaseModelAdapter):
 
     def __init__(self, custom_model: CustomModel, **kwargs):

evalscope/models/local_model.py

@@ -3,6 +3,7 @@ from typing import TYPE_CHECKING, Optional
 
 from evalscope.constants import DEFAULT_MODEL_CACHE_DIR, DEFAULT_MODEL_REVISION, EvalType
 from evalscope.utils.logger import get_logger
+from evalscope.utils.model_utils import get_device
 
 if TYPE_CHECKING:
     from evalscope.config import TaskConfig
@@ -28,7 +29,7 @@ class LocalModel:
 
         self.model_id = model_id
         self.model_revision = model_revision
-        self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+        self.device = device_map
 
         self.tokenizer = AutoTokenizer.from_pretrained(
             self.model_id,
@@ -64,7 +65,7 @@ def get_local_model(task_cfg: 'TaskConfig') -> Optional[LocalModel]:
     if task_cfg.eval_type != EvalType.CHECKPOINT:
         return None
     else:
-        device_map = task_cfg.model_args.get('device_map', 'auto')
+        device_map = task_cfg.model_args.get('device_map', get_device())
         cache_dir = task_cfg.model_args.get('cache_dir', None)
         model_precision = task_cfg.model_args.get('precision', 'torch.float16')
         model_revision = task_cfg.model_args.get('revision', DEFAULT_MODEL_REVISION)

evalscope/models/register.py

@@ -0,0 +1,28 @@
+MODEL_ADAPTERS = {}
+
+
+def register_model_adapter(name):
+    """
+    Decorator to register a model adapter with a given name.
+    :param name: The name of the model adapter.
+    """
+
+    def decorator(adapter):
+        if name in MODEL_ADAPTERS:
+            raise ValueError(f"Model adapter '{name}' is already registered.")
+        MODEL_ADAPTERS[name] = adapter
+        return adapter
+
+    return decorator
+
+
+def get_model_adapter(name):
+    """
+    Retrieve a registered model adapter by name.
+    :param name: The name of the model adapter.
+    :return: The model adapter class or function.
+    """
+    if name not in MODEL_ADAPTERS:
+        raise ValueError(
+            f"Model adapter '{name}' is not registered. Available model adapters: {list(MODEL_ADAPTERS.keys())}")
+    return MODEL_ADAPTERS[name]

evalscope/models/server_adapter.py

@@ -1,14 +1,18 @@
-import requests
-import time
+import openai
+from collections import defaultdict
+from inspect import signature
+from openai.types.chat import ChatCompletion, ChatCompletionChunk
+from openai.types.chat.chat_completion import ChatCompletionMessage, Choice
 from typing import List, Optional, Union
 
 from evalscope.models.base_adapter import BaseModelAdapter
-from evalscope.utils.chat_service import ChatMessage
+from evalscope.models.register import register_model_adapter
 from evalscope.utils.logger import get_logger
 
 logger = get_logger()
 
 
+@register_model_adapter('server')
 class ServerModelAdapter(BaseModelAdapter):
     """
     Server model adapter to request remote API model and generate results.
@@ -21,19 +25,32 @@ class ServerModelAdapter(BaseModelAdapter):
             model_id: The ID of the remote API model.
             api_key: The API key of the remote API model.
         """
-        self.api_url = api_url
+        self.api_url = api_url.rstrip('/').rsplit('/chat/completions', 1)[0]
         self.model_id = model_id
         self.api_key = api_key
+
+        self.client = openai.OpenAI(
+            api_key=api_key,
+            base_url=self.api_url,
+        )
+        self.supported_params = self._get_supported_params()
+
         self.seed = kwargs.get('seed', None)
+        self.timeout = kwargs.get('timeout', 60)
+        self.stream = kwargs.get('stream', False)
         self.model_cfg = {'api_url': api_url, 'model_id': model_id, 'api_key': api_key}
         super().__init__(model=None, model_cfg=self.model_cfg, **kwargs)
 
-    def predict(self, inputs: List[Union[str, dict, list]], infer_cfg: dict = None) -> List[dict]:
+    def _get_supported_params(self):
+        sig = signature(self.client.chat.completions.create)
+        return list(sig.parameters.keys())
+
+    def predict(self, inputs: List[dict], infer_cfg: dict = None) -> List[dict]:
         """
         Model prediction func.
 
         Args:
-            inputs (List[Union[str, dict, list]]): The input data.
+            inputs (List[dict]): The input data.
             infer_cfg (dict): Inference configuration.
 
         Returns:
@@ -63,20 +80,19 @@ class ServerModelAdapter(BaseModelAdapter):
         response = self.send_request(request_json)
         return response
 
-    def make_request_content(self, query: str, system_prompt: Optional[str] = None) -> dict:
+    def make_request_content(self, query: str, system_prompt: Optional[str] = None) -> list:
         """
-        Make request content for API.
+        Make request content for OpenAI API.
         """
+        messages = []
         if system_prompt:
-            messages = [
-                ChatMessage(role='system', content=system_prompt).model_dump(exclude_unset=True),
-                ChatMessage(role='user', content=query).model_dump(exclude_unset=True)
-            ]
-        else:
-            messages = [ChatMessage(role='user', content=query).model_dump(exclude_unset=True)]
-        return {'messages': messages}
+            messages.append({'role': 'system', 'content': system_prompt})
+
+        messages.append({'role': 'user', 'content': query})
 
-    def make_request(self, content: dict, infer_cfg: dict = {}) -> dict:
+        return messages
+
+    def make_request(self, content: list, infer_cfg: dict = {}) -> dict:
         """Make request to remote API."""
         # Format request JSON according to OpenAI API format
         from evalscope.config import DEFAULT_GENERATION_CONFIG
@@ -86,20 +102,82 @@ class ServerModelAdapter(BaseModelAdapter):
                 'temperature': 0.0,
             }
 
-        request_json = {'model': self.model_id, **content, **infer_cfg}
+        request_json = {'model': self.model_id, 'messages': content, **infer_cfg}
+
+        if self.timeout:
+            request_json['timeout'] = self.timeout
+
+        if self.stream:
+            request_json['stream'] = self.stream
+            request_json['stream_options'] = {'include_usage': True}
+
         logger.debug(f'Request to remote API: {request_json}')
+
         return request_json
 
-    def send_request(self, request_json: dict, max_retries: int = 3) -> dict:
-        for attempt in range(max_retries):
-            response = requests.post(
-                self.api_url, json=request_json, headers={'Authorization': f'Bearer {self.api_key}'})
-            if response.status_code == 200:
-                response_data = response.json()
-                return response_data
-            logger.warning(f'Failed to request to remote API: {response.status_code} {response.text}')
-            if attempt < max_retries - 1:
-                time.sleep(5)  # Sleep for 5 seconds before retrying
+    def _parse_extra_params(self, request_json):
+        api_params = {}
+        extra_body = {}
+        for key, value in request_json.items():
+            if key in self.supported_params:
+                api_params[key] = value
             else:
-                raise RuntimeError(f'Failed to request to remote API after {max_retries} attempts: '
-                                   f'{response.status_code} {response.text}')
+                extra_body[key] = value
+
+        if extra_body:
+            api_params['extra_body'] = extra_body
+        return api_params
+
+    def send_request(self, request_json: dict) -> dict:
+        try:
+            parsed_request = self._parse_extra_params(request_json)
+            response = self.client.chat.completions.create(**parsed_request)
+
+            if response and self.stream:
+                response = self._collect_stream_response(response)
+
+            return response.model_dump(exclude_unset=True)
+        except Exception as e:
+            logger.error(f'Error when calling remote API: {str(e)}')
+            raise
+
+    def _collect_stream_response(self, response_stream: List[ChatCompletionChunk]) -> ChatCompletion:
+        collected_chunks = []
+        collected_messages = defaultdict(list)
+        collected_reasoning = defaultdict(list)
+
+        for chunk in response_stream:
+            collected_chunks.append(chunk)
+            for choice in chunk.choices:
+                if hasattr(choice.delta, 'reasoning_content') and choice.delta.reasoning_content is not None:
+                    collected_reasoning[choice.index].append(choice.delta.reasoning_content)
+                if choice.delta.content is not None:
+                    collected_messages[choice.index].append(choice.delta.content)
+
+        choices = []
+        for index, messages in collected_messages.items():
+            full_reply_content = ''.join(messages)
+            reasoning = ''.join(collected_reasoning[index])
+            # use the finish_reason from the last chunk that generated this choice
+            finish_reason = None
+            for chunk in reversed(collected_chunks):
+                if chunk.choices and chunk.choices[0].index == index:
+                    finish_reason = chunk.choices[0].finish_reason
+                    break
+
+            choice = Choice(
+                finish_reason=finish_reason or 'stop',
+                index=index,
+                message=ChatCompletionMessage(
+                    role='assistant', content=full_reply_content, reasoning_content=reasoning))
+            choices.append(choice)
+
+        # build the final completion object
+        return ChatCompletion(
+            id=collected_chunks[0].id,
+            choices=choices,
+            created=collected_chunks[0].created,
+            model=collected_chunks[0].model,
+            object='chat.completion',
+            usage=collected_chunks[-1].usage  # use the usage from the last chunk
+        )

evalscope/perf/__init__.py

@@ -1 +0,0 @@
-from evalscope.perf.main import run_perf_benchmark

evalscope/perf/arguments.py

@@ -21,9 +21,9 @@ class Arguments:
     # Connection settings
     url: str = 'http://127.0.0.1:8877/v1/chat/completions'  # URL for the API connection
     headers: Dict[str, Any] = field(default_factory=dict)  # Custom headers
-    connect_timeout: int = 120  # Connection timeout in seconds
-    read_timeout: int = 120  # Read timeout in seconds
-    api_key: str = 'EMPTY'
+    connect_timeout: int = 600  # Connection timeout in seconds
+    read_timeout: int = 600  # Read timeout in seconds
+    api_key: Optional[str] = None
 
     # Performance and parallelism
     number: Optional[int] = None  # Number of requests to be made
@@ -61,6 +61,7 @@ class Arguments:
     stream: Optional[bool] = None  # Whether to stream the response
     temperature: Optional[float] = None  # Temperature setting for the response
     top_p: Optional[float] = None  # Top-p (nucleus) sampling setting for the response
+    top_k: Optional[int] = None  # Top-k sampling setting for the response
 
     @staticmethod
     def from_args(args):
@@ -99,7 +100,9 @@ class Arguments:
             stop_token_ids=args.stop_token_ids,
             stream=args.stream,
             temperature=args.temperature,
-            top_p=args.top_p)
+            top_p=args.top_p,
+            top_k=args.top_k,
+        )
 
     def __post_init__(self):
         self.headers = self.headers or {}  # Default to empty dictionary
@@ -122,7 +125,13 @@ class ParseKVAction(argparse.Action):
             setattr(namespace, self.dest, {})
         else:
             try:
-                kv_dict = dict(kv.split('=') for kv in values)
+                kv_dict = {}
+                for kv in values:
+                    parts = kv.split('=', 1)  # only split the first '='
+                    if len(parts) != 2:
+                        raise ValueError(f'Invalid key-value pair: {kv}')
+                    key, value = parts
+                    kv_dict[key.strip()] = value.strip()
                 setattr(namespace, self.dest, kv_dict)
             except ValueError as e:
                 parser.error(f'Error parsing key-value pairs: {e}')
@@ -141,9 +150,9 @@ def add_argument(parser: argparse.ArgumentParser):
     parser.add_argument('--url', type=str, default='http://127.0.0.1:8877/v1/chat/completions')
     parser.add_argument('--port', type=int, default=8877, help='The port for local inference')
     parser.add_argument('--headers', nargs='+', dest='headers', action=ParseKVAction, help='Extra HTTP headers')
-    parser.add_argument('--api-key', type=str, required=False, default='EMPTY', help='The API key for authentication')
-    parser.add_argument('--connect-timeout', type=int, default=120, help='The network connection timeout')
-    parser.add_argument('--read-timeout', type=int, default=120, help='The network read timeout')
+    parser.add_argument('--api-key', type=str, required=False, default=None, help='The API key for authentication')
+    parser.add_argument('--connect-timeout', type=int, default=600, help='The network connection timeout')
+    parser.add_argument('--read-timeout', type=int, default=600, help='The network read timeout')
 
     # Performance and parallelism
     parser.add_argument('-n', '--number', type=int, default=None, help='How many requests to be made')
@@ -183,6 +192,7 @@ def add_argument(parser: argparse.ArgumentParser):
     parser.add_argument('--stream', action='store_true', help='Stream output with SSE', default=None)
     parser.add_argument('--temperature', type=float, help='The sample temperature', default=None)
     parser.add_argument('--top-p', type=float, help='Sampling top p', default=None)
+    parser.add_argument('--top-k', type=int, help='Sampling top k', default=None)
 
     # yapf: enable
 

evalscope/perf/http_client.py

@@ -23,10 +23,7 @@ class AioHttpClient:
         self.read_timeout = args.read_timeout
         self.connect_timeout = args.connect_timeout
         self.client = aiohttp.ClientSession(
-            timeout=aiohttp.ClientTimeout(
-                total=self.read_timeout + self.connect_timeout,
-                connect=self.connect_timeout,
-                sock_read=self.read_timeout),
+            timeout=aiohttp.ClientTimeout(connect=self.connect_timeout, sock_read=self.read_timeout),
             connector=aiohttp.TCPConnector(limit=1),
             trace_configs=[self._create_trace_config()] if args.debug else [])
 
@@ -102,6 +99,11 @@ class AioHttpClient:
             async with self.client.request('POST', url=self.url, data=data, headers=headers) as response:
                 async for rsp in self._handle_response(response):
                     yield rsp
+        except asyncio.TimeoutError:
+            logger.error(
+                f'TimeoutError: connect_timeout: {self.connect_timeout}, read_timeout: {self.read_timeout}. Please set longger timeout.'  # noqa: E501
+            )
+            yield (True, None, 'Timeout')
         except (aiohttp.ClientConnectorError, Exception) as e:
             logger.error(e)
             yield (True, None, e)
@@ -143,9 +145,9 @@ async def test_connection(args: Arguments) -> bool:
         client = AioHttpClient(args)
         async with client:
             if 'chat/completions' in args.url:
-                request = {'messages': [{'role': 'user', 'content': 'hello'}], 'model': args.model}
+                request = {'messages': [{'role': 'user', 'content': 'hello'}], 'model': args.model, 'max_tokens': 10}
             else:
-                request = {'prompt': 'hello', 'model': args.model}
+                request = {'prompt': 'hello', 'model': args.model, 'max_tokens': 10}
             async for is_error, state_code, response_data in client.post(request):
                 return is_error, state_code, response_data
 

evalscope/perf/plugin/api/openai_api.py

@@ -92,6 +92,8 @@ class OpenaiPlugin(ApiPluginBase):
             payload['temperature'] = param.temperature
         if param.top_p is not None:
             payload['top_p'] = param.top_p
+        if param.top_k is not None:
+            payload['top_k'] = param.top_k
         return payload
 
     def parse_responses(self, responses, request: Any = None, **kwargs) -> Dict:
@@ -155,5 +157,13 @@ class OpenaiPlugin(ApiPluginBase):
                 input_tokens += len(self.tokenizer.encode(request['messages'][0]['content']))
                 output_tokens += len(self.tokenizer.encode(full_response_content))
         else:
-            logger.warning('No usage information found. Please specify `--tokenizer-path` to generate usage details.')
+            raise ValueError('Error: Unable to retrieve usage information\n\n'
+                             'This error occurs when:\n'
+                             '1. The API response does not contain usage data, AND\n'
+                             '2. No tokenizer has been specified or found.\n\n'
+                             'To resolve this issue, do ONE of the following:\n'
+                             "a) Ensure that the API you're using supports and returns usage information, OR\n"
+                             'b) Specify a tokenizer using the `--tokenizer-path` parameter.\n\n'
+                             'If you continue to experience issues, '
+                             'please open an issue on our GitHub repository https://github.com/modelscope/evalscope .')
         return input_tokens, output_tokens

evalscope/perf/utils/analysis_result.py

@@ -3,7 +3,7 @@ import json
 import pickle
 import sqlite3
 
-result_db_path = '/mnt/data/data/user/maoyunlin.myl/eval-scope/outputs/qwen2.5_benchmark_20241111_160543.db'
+result_db_path = './outputs/qwen2.5_benchmark_20241111_160543.db'
 con = sqlite3.connect(result_db_path)
 query_sql = "SELECT request, response_messages, prompt_tokens, completion_tokens \
                 FROM result WHERE success='1'"

evalscope/perf/utils/benchmark_util.py

@@ -23,6 +23,7 @@ class BenchmarkData:
     n_chunks: int = 0
     n_chunks_time: float = 0.0
     max_gpu_memory_cost = 0
+    time_per_output_token: float = 0.0
 
     prompt_tokens = None
     completion_tokens = None
@@ -37,6 +38,7 @@ class BenchmarkData:
             self.first_chunk_latency = self.query_latency
             self.n_chunks = 1
             self.n_chunks_time = self.query_latency
+        self.time_per_output_token = self.query_latency / self.completion_tokens
 
     def _calculate_tokens(self, api_plugin):
         self.prompt_tokens, self.completion_tokens = \
@@ -63,6 +65,7 @@ class BenchmarkMetrics:
     start_time: Optional[float] = None
     total_time: float = 1.0
     n_total_queries: int = 0
+    n_time_per_output_token: float = 0.0
 
     avg_first_chunk_latency: float = -1
     avg_latency: float = -1
@@ -92,6 +95,7 @@ class BenchmarkMetrics:
             self.total_first_chunk_latency += benchmark_data.first_chunk_latency
             self.n_total_chunks += benchmark_data.n_chunks
             self.total_chunks_time += benchmark_data.n_chunks_time
+            self.n_time_per_output_token += benchmark_data.time_per_output_token
         else:
             self.n_failed_queries += 1
 
@@ -108,7 +112,7 @@ class BenchmarkMetrics:
             self.avg_prompt_tokens = self.n_total_prompt_tokens / self.n_succeed_queries
             self.avg_completion_tokens = self.n_total_completion_tokens / self.n_succeed_queries
             self.avg_token_per_seconds = self.n_total_completion_tokens / self.total_time
-            self.avg_time_per_token = self.total_time / self.n_total_completion_tokens
+            self.avg_time_per_token = self.n_time_per_output_token / self.n_succeed_queries
             self.qps = self.n_succeed_queries / self.total_time
         except ZeroDivisionError as e:
             logger.exception(e)
@@ -125,7 +129,7 @@ class BenchmarkMetrics:
             'Average QPS': round(self.qps, default_ndigits),
             'Average latency (s)': round(self.avg_latency, default_ndigits),
             'Average time to first token (s)': round(self.avg_first_chunk_latency, default_ndigits),
-            'Average time per output token (s)': round(self.avg_time_per_token, 5),
+            'Average time per output token (s)': round(self.avg_time_per_token, default_ndigits),
             'Average input tokens per request': round(self.avg_prompt_tokens, default_ndigits),
             'Average output tokens per request': round(self.avg_completion_tokens, default_ndigits),
             'Average package latency (s)': round(self.avg_chunk_time, default_ndigits),

evalscope/report/app.py

@@ -19,6 +19,9 @@ from evalscope.version import __version__
 logger = get_logger()
 
 PLOTLY_THEME = 'plotly_dark'
+REPORT_TOKEN = '@@'
+MODEL_TOKEN = '::'
+DATASET_TOKEN = ', '
 
 
 def scan_for_report_folders(root_path):
@@ -42,8 +45,9 @@ def scan_for_report_folders(root_path):
             datasets = []
             for dataset_item in glob.glob(os.path.join(model_item, '*.json')):
                 datasets.append(os.path.basename(dataset_item).split('.')[0])
-            datasets = ','.join(datasets)
-            reports.append(f'{os.path.basename(folder)}@{os.path.basename(model_item)}:{datasets}')
+            datasets = DATASET_TOKEN.join(datasets)
+            reports.append(
+                f'{os.path.basename(folder)}{REPORT_TOKEN}{os.path.basename(model_item)}{MODEL_TOKEN}{datasets}')
 
     reports = sorted(reports, reverse=True)
     logger.debug(f'reports: {reports}')
@@ -51,9 +55,9 @@ def scan_for_report_folders(root_path):
 
 
 def process_report_name(report_name: str):
-    prefix, report_name = report_name.split('@')
-    model_name, datasets = report_name.split(':')
-    datasets = datasets.split(',')
+    prefix, report_name = report_name.split(REPORT_TOKEN)
+    model_name, datasets = report_name.split(MODEL_TOKEN)
+    datasets = datasets.split(DATASET_TOKEN)
     return prefix, model_name, datasets
 
 
@@ -121,6 +125,9 @@ def get_compare_report_df(acc_df: pd.DataFrame):
 
 
 def plot_single_report_scores(df: pd.DataFrame):
+    if df is None:
+        return None
+    logger.debug(f'df: {df}')
     plot = px.bar(df, x=df[ReportKey.dataset_name], y=df[ReportKey.score], text=df[ReportKey.score])
 
     width = 0.2 if len(df[ReportKey.dataset_name]) <= 5 else None
@@ -171,7 +178,7 @@ def plot_single_dataset_scores(df: pd.DataFrame):
         text=df[ReportKey.score],
         barmode='group')
 
-    width = 0.2 if len(df[ReportKey.subset_name]) <= 5 else None
+    width = 0.2 if len(df[ReportKey.subset_name]) <= 3 else None
     plot.update_traces(width=width, texttemplate='%{text:.2f}', textposition='outside')
     plot.update_layout(uniformtext_minsize=12, uniformtext_mode='hide', yaxis=dict(range=[0, 1]), template=PLOTLY_THEME)
     return plot
@@ -519,8 +526,8 @@ def create_single_model_tab(sidebar: SidebarComponents, lang: str):
         outputs=[report_list, task_config, dataset_radio, work_dir, model_name])
     def update_single_report_data(root_path, report_name):
         report_list, datasets, task_cfg = load_single_report(root_path, report_name)
-        work_dir = os.path.join(root_path, report_name.split('@')[0])
-        model_name = report_name.split('@')[1].split(':')[0]
+        work_dir = os.path.join(root_path, report_name.split(REPORT_TOKEN)[0])
+        model_name = report_name.split(REPORT_TOKEN)[1].split(MODEL_TOKEN)[0]
         return (report_list, task_cfg, gr.update(choices=datasets, value=datasets[0]), work_dir, model_name)
 
     @report_list.change(inputs=[report_list], outputs=[score_plot, score_table, sunburst_plot])