evalscope 0.8.0__py3-none-any.whl → 0.10.1__py3-none-any.whl

evalscope/models/local_model.py

@@ -0,0 +1,74 @@
+import torch
+from typing import TYPE_CHECKING, Optional
+
+from evalscope.constants import DEFAULT_MODEL_CACHE_DIR, DEFAULT_MODEL_REVISION, EvalType
+from evalscope.utils.logger import get_logger
+
+if TYPE_CHECKING:
+    from evalscope.config import TaskConfig
+
+logger = get_logger()
+
+
+class LocalModel:
+
+    def __init__(self,
+                 model_id: str,
+                 model_revision: str = DEFAULT_MODEL_REVISION,
+                 device_map: str = 'auto',
+                 torch_dtype: str = 'auto',
+                 cache_dir: str = None,
+                 **kwargs):
+        from modelscope import AutoModelForCausalLM, AutoTokenizer
+
+        model_cache_dir = cache_dir or DEFAULT_MODEL_CACHE_DIR
+
+        if isinstance(torch_dtype, str) and torch_dtype != 'auto':
+            torch_dtype = eval(torch_dtype)
+
+        self.model_id = model_id
+        self.model_revision = model_revision
+        self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+
+        self.tokenizer = AutoTokenizer.from_pretrained(
+            self.model_id,
+            revision=model_revision,
+            trust_remote_code=True,
+            cache_dir=model_cache_dir,
+        )
+
+        self.model = AutoModelForCausalLM.from_pretrained(
+            self.model_id,
+            revision=model_revision,
+            device_map=device_map,
+            trust_remote_code=True,
+            torch_dtype=torch_dtype,
+            cache_dir=model_cache_dir,
+        )
+
+        self.model_cfg = {
+            'model_id': model_id,
+            'device_map': device_map,
+            'torch_dtype': str(torch_dtype),
+        }
+
+
+def get_local_model(task_cfg: 'TaskConfig') -> Optional[LocalModel]:
+    """Get the base local model for the task. If the task is not checkpoint-based, return None.
+       Avoids loading model multiple times for different datasets.
+    """
+    if task_cfg.eval_type != EvalType.CHECKPOINT:
+        return None
+    else:
+        device_map = task_cfg.model_args.get('device_map', 'auto')
+        cache_dir = task_cfg.model_args.get('cache_dir', None)
+        model_precision = task_cfg.model_args.get('precision', 'torch.float16')
+        model_revision = task_cfg.model_args.get('revision', DEFAULT_MODEL_REVISION)
+
+        base_model = LocalModel(
+            model_id=task_cfg.model,
+            model_revision=model_revision,
+            device_map=device_map,
+            torch_dtype=model_precision,
+            cache_dir=cache_dir)
+        return base_model

evalscope/models/model.py

@@ -1,7 +1,14 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
+import os
+import random
+import time
 from abc import ABC, abstractmethod
 from typing import Any
 
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
 
 class BaseModel(ABC):
 
@@ -86,3 +93,137 @@ class ChatBaseModel(BaseModel):
                 }
         """
         raise NotImplementedError
+
+
+class OpenAIModel(ChatBaseModel):
+    """
+    APIs of OpenAI models.
+    Available models: gpt-3.5-turbo, gpt-4
+    """
+
+    MAX_RETRIES = 3
+
+    def __init__(self, model_cfg: dict, **kwargs):
+        super(OpenAIModel, self).__init__(model_cfg=model_cfg, **kwargs)
+
+        openai_api_key = os.environ.get('OPENAI_API_KEY', None)
+        self.api_key = self.model_cfg.get('api_key', openai_api_key)
+
+        if not self.api_key:
+            logger.error('OpenAI API key is not provided, please set it in environment variable OPENAI_API_KEY')
+            # raise ValueError(
+            #     'OpenAI API key is not provided, '
+            #     'please set it in environment variable OPENAI_API_KEY')
+
+    def predict(self, model_id: str, inputs: dict, **kwargs) -> dict:
+
+        sys_prompt: str = inputs.get('sys_prompt', '')
+        user_prompt: str = inputs.get('user_prompt', '')
+
+        # model_id: str = kwargs.get('model_id', '')
+        temperature: float = kwargs.pop('temperature', 0.2)
+        max_tokens: int = kwargs.pop('max_tokens', 1024)
+        mode: str = kwargs.pop('mode', 'chat.completion')
+
+        logger.info(f'Using OpenAI model_id: {model_id}')
+
+        res = self._predict(
+            model_id=model_id,
+            sys_prompt=sys_prompt,
+            user_prompt=user_prompt,
+            temperature=temperature,
+            max_tokens=max_tokens,
+            mode=mode)
+
+        return res
+
+    def _predict(
+        self,
+        model_id,
+        sys_prompt,
+        user_prompt,
+        temperature,
+        max_tokens,
+        mode: str = 'chat.completion',
+    ) -> dict:
+        import openai
+
+        res = {}
+        openai.api_key = self.api_key
+
+        for i in range(self.MAX_RETRIES):
+            try:
+                if mode == 'chat.completion':
+                    resp = openai.ChatCompletion.create(
+                        model=model_id,
+                        messages=[{
+                            'role': 'system',
+                            'content': sys_prompt
+                        }, {
+                            'role': 'user',
+                            'content': user_prompt
+                        }],
+                        temperature=temperature,
+                        max_tokens=max_tokens)
+
+                    if resp:
+                        ans_text = resp['choices'][0]['message']['content']
+                        model_id = resp['model']
+                    else:
+                        logger.warning(f'OpenAI GPT API call failed: got empty response '
+                                       f'for input {sys_prompt} {user_prompt}')
+                        ans_text = ''
+                        model_id = ''
+
+                    res['ans_text'] = ans_text
+                    res['model_id'] = model_id
+                else:
+                    raise ValueError(f'Invalid mode: {mode}')
+
+                return res
+
+            except Exception as e:
+                logger.warning(f'OpenAI API call failed: {e}')
+                time.sleep(3)
+        logger.error(f'OpenAI API call failed after {self.MAX_RETRIES} retries')
+        return res
+
+
+class DummyChatModel(ChatBaseModel):
+
+    MODEL_ID = 'dummy_chat_model_0801'
+    REVISION = 'v1.0.0'
+
+    def __init__(self, model_cfg: dict, **kwargs):
+        model_cfg['model_id'] = self.MODEL_ID
+        model_cfg['revision'] = self.REVISION
+        super(DummyChatModel, self).__init__(model_cfg=model_cfg)
+
+    def predict(self, inputs: dict, **kwargs) -> dict:
+
+        debug: bool = False
+        if debug:
+            messages = inputs['messages']
+            history = inputs['history']
+
+            logger.info(f'** messages: {messages}')
+            logger.info(f'** history: {history}')
+
+        choice = random.choice(['A', 'B', 'C', 'D'])
+
+        # Build response
+        res = {
+            'choices': [{
+                'index': 0,
+                'message': {
+                    'content': choice,
+                    'role': 'assistant'
+                }
+            }],
+            'created': time.time(),
+            'model': self.MODEL_ID + '-' + self.REVISION,
+            'object': 'chat.completion',
+            'usage': {}
+        }
+
+        return res

evalscope/models/server_adapter.py

@@ -0,0 +1,111 @@
+import requests
+import time
+from typing import Optional, Union
+
+from evalscope.models.base_adapter import BaseModelAdapter
+from evalscope.utils.chat_service import ChatMessage
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+
+class ServerModelAdapter(BaseModelAdapter):
+    """
+    Server model adapter to request remote API model and generate results.
+    """
+
+    def __init__(self, api_url: str, model_id: str, api_key: str = 'EMPTY', **kwargs):
+        """
+        Args:
+            api_url: The URL of the remote API model.
+            model_id: The ID of the remote API model.
+            api_key: The API key of the remote API model.
+        """
+        self.api_url = api_url
+        self.model_id = model_id
+        self.api_key = api_key
+        self.seed = kwargs.get('seed', None)
+        self.model_cfg = {'api_url': api_url, 'model_id': model_id, 'api_key': api_key}
+        super().__init__(model=None, model_cfg=self.model_cfg, **kwargs)
+
+    def predict(self, inputs: Union[str, dict, list], infer_cfg: dict = None) -> dict:
+        """
+        Model prediction func.
+
+        Args:
+            inputs (Union[str, dict, list]): The input data.
+            infer_cfg (dict): Inference configuration.
+
+        Returns:
+            res (dict): The model prediction results.
+        """
+        infer_cfg = infer_cfg or {}
+
+        # Process inputs
+        if isinstance(inputs, str):
+            query = inputs
+            system_prompt = None
+        elif isinstance(inputs, dict):
+            data: list = inputs['data']
+            if isinstance(data[0], tuple):  # for truthful_qa and hellaswag
+                query = '\n'.join(''.join(item) for item in data)
+                system_prompt = inputs.get('system_prompt', None)
+            else:
+                query = data[0]
+                system_prompt = inputs.get('system_prompt', None)
+        elif isinstance(inputs, list):
+            query = '\n'.join(inputs)
+            system_prompt = None
+        else:
+            raise TypeError(f'Unsupported inputs type: {type(inputs)}')
+
+        content = self.make_request_content(query, system_prompt)
+        request_json = self.make_request(content, infer_cfg)
+        response = self.send_request(request_json)
+        return response
+
+    def make_request_content(self, query: str, system_prompt: Optional[str] = None) -> dict:
+        """
+        Make request content for API.
+        """
+        if system_prompt is not None:
+            messages = [
+                ChatMessage(role='system', content=system_prompt).model_dump(exclude_unset=True),
+                ChatMessage(role='user', content=query).model_dump(exclude_unset=True)
+            ]
+        else:
+            messages = [ChatMessage(role='user', content=query).model_dump(exclude_unset=True)]
+        return {'messages': messages}
+
+    def make_request(self, content: dict, infer_cfg: dict = {}) -> dict:
+        """Make request to remote API."""
+        # Format request JSON according to OpenAI API format
+        do_sample = infer_cfg.get('do_sample', False)
+        temperature = infer_cfg.get('temperature', 0.0) if do_sample else 0.0
+
+        request_json = {
+            **content, 'model': self.model_id,
+            'max_tokens': infer_cfg.get('max_tokens', 2048),
+            'temperature': temperature,
+            'top_p': infer_cfg.get('top_p', 1.0),
+            'n': infer_cfg.get('num_return_sequences', 1),
+            'stop': infer_cfg.get('stop', None)
+        }
+        if self.seed is not None:
+            request_json['seed'] = self.seed
+        logger.debug(f'Request to remote API: {request_json}')
+        return request_json
+
+    def send_request(self, request_json: dict, max_retries: int = 3) -> dict:
+        for attempt in range(max_retries):
+            response = requests.post(
+                self.api_url, json=request_json, headers={'Authorization': f'Bearer {self.api_key}'})
+            if response.status_code == 200:
+                response_data = response.json()
+                return response_data
+            logger.warning(f'Failed to request to remote API: {response.status_code} {response.text}')
+            if attempt < max_retries - 1:
+                time.sleep(5)  # Sleep for 5 seconds before retrying
+            else:
+                raise RuntimeError(f'Failed to request to remote API after {max_retries} attempts: '
+                                   f'{response.status_code} {response.text}')

evalscope/perf/__init__.py

@@ -0,0 +1 @@
+from evalscope.perf.main import run_perf_benchmark

evalscope/perf/arguments.py

@@ -16,7 +16,7 @@ class Arguments:
     attn_implementation: Optional[str] = None  # Attention implementaion, only for local inference
     api: str = 'openai'  # API to be used (default: 'openai')
     tokenizer_path: Optional[str] = None  # Path to the tokenizer
-    port: str = '8877'  # Port number for the local API server
+    port: int = 8877  # Port number for the local API server
 
     # Connection settings
     url: str = 'http://127.0.0.1:8877/v1/chat/completions'  # URL for the API connection
@@ -68,6 +68,7 @@ class Arguments:
             model=args.model,
             attn_implementation=args.attn_implementation,
             url=args.url,
+            port=args.port,
             api_key=args.api_key,
             connect_timeout=args.connect_timeout,
             read_timeout=args.read_timeout,
@@ -138,6 +139,7 @@ def add_argument(parser: argparse.ArgumentParser):
 
     # Connection settings
     parser.add_argument('--url', type=str, default='http://127.0.0.1:8877/v1/chat/completions')
+    parser.add_argument('--port', type=int, default=8877, help='The port for local inference')
     parser.add_argument('--headers', nargs='+', dest='headers', action=ParseKVAction, help='Extra HTTP headers')
     parser.add_argument('--api-key', type=str, required=False, default='EMPTY', help='The API key for authentication')
     parser.add_argument('--connect-timeout', type=int, default=120, help='The network connection timeout')

evalscope/perf/benchmark.py

@@ -157,7 +157,7 @@ async def statistic_benchmark_metric_worker(benchmark_data_queue: asyncio.Queue,
             while not (data_process_completed_event.is_set() and benchmark_data_queue.empty()):
                 try:
                     # Attempt to get benchmark data from the queue with a timeout
-                    benchmark_data = await asyncio.wait_for(benchmark_data_queue.get(), timeout=1)
+                    benchmark_data = await asyncio.wait_for(benchmark_data_queue.get(), timeout=0.01)
                     benchmark_data_queue.task_done()
                 except asyncio.TimeoutError:
                     # If timeout, continue to the next iteration
@@ -195,9 +195,9 @@ async def start_server(args: Arguments) -> bool:
         server.start()
 
         if args.dataset.startswith('speed_benchmark'):
-            args.url = 'http://127.0.0.1:8877/v1/completions'
+            args.url = f'http://127.0.0.1:{args.port}/v1/completions'
         else:
-            args.url = 'http://127.0.0.1:8877/v1/chat/completions'
+            args.url = f'http://127.0.0.1:{args.port}/v1/chat/completions'
 
     if not await test_connection(args):
         raise TimeoutError('Test connection failed')

evalscope/perf/main.py

@@ -1,5 +1,4 @@
 import asyncio
-import logging
 import os
 import platform
 from argparse import Namespace
@@ -8,7 +7,7 @@ from evalscope.perf.arguments import Arguments, parse_args
 from evalscope.perf.benchmark import benchmark
 from evalscope.perf.utils.db_util import get_output_path
 from evalscope.perf.utils.handler import add_signal_handlers
-from evalscope.utils.logger import get_logger
+from evalscope.utils.logger import configure_logging, get_logger
 from evalscope.utils.utils import seed_everything
 
 logger = get_logger()
@@ -19,14 +18,13 @@ def run_perf_benchmark(args):
         args = Arguments(**args)
     elif isinstance(args, Namespace):
         args = Arguments.from_args(args)
-    seed_everything(args.seed)
+
+    if args.seed is not None:
+        seed_everything(args.seed)
 
     # Setup logger and output
     args.outputs_dir = get_output_path(args)
-    get_logger(log_file=os.path.join(args.outputs_dir, 'benchmark.log'), force=True)
-
-    if args.debug:
-        get_logger(log_level=logging.DEBUG, force=True)
+    configure_logging(args.debug, os.path.join(args.outputs_dir, 'benchmark.log'))
 
     logger.info('Starting benchmark...')
     logger.info(args)

evalscope/perf/plugin/api/custom_api.py

@@ -1,5 +1,4 @@
 import json
-from transformers import AutoTokenizer
 from typing import Any, Dict, Iterator, List
 
 from evalscope.perf.arguments import Arguments
@@ -25,6 +24,7 @@ class CustomPlugin(ApiPluginBase):
         """
         super().__init__(model_path=mode_path)
         if mode_path is not None:
+            from transformers import AutoTokenizer
             self.tokenizer = AutoTokenizer.from_pretrained(mode_path)
         else:
             self.tokenizer = None

evalscope/perf/plugin/api/openai_api.py

@@ -1,7 +1,6 @@
 import json
 import os
-from transformers import AutoTokenizer
-from typing import Any, Dict, Iterator, List
+from typing import Any, Dict, Iterator, List, Union
 
 from evalscope.perf.arguments import Arguments
 from evalscope.perf.plugin.api.base import ApiPluginBase
@@ -25,11 +24,12 @@ class OpenaiPlugin(ApiPluginBase):
         """
         super().__init__(model_path=mode_path)
         if mode_path is not None:
+            from transformers import AutoTokenizer
             self.tokenizer = AutoTokenizer.from_pretrained(mode_path)
         else:
             self.tokenizer = None
 
-    def build_request(self, messages: List[Dict] | str, param: Arguments) -> Dict:
+    def build_request(self, messages: Union[List[Dict], str], param: Arguments) -> Dict:
         """Build the openai format request based on prompt, dataset
 
         Args:
@@ -96,60 +96,64 @@ class OpenaiPlugin(ApiPluginBase):
 
     def parse_responses(self, responses, request: Any = None, **kwargs) -> Dict:
         """Parser responses and return number of request and response tokens.
-           sample of the output delta:
-           {"id":"4","object":"chat.completion.chunk","created":1714030870,"model":"llama3","choices":[{"index":0,"delta":{"role":"assistant","content":""},"logprobs":null,"finish_reason":null}]}
+        Only one response for non-stream, multiple responses for stream.
+        """
 
+        # when stream, the last response is the full usage
+        # when non-stream, the last response is the first response
+        last_response_js = json.loads(responses[-1])
+        if 'usage' in last_response_js and last_response_js['usage']:
+            input_tokens = last_response_js['usage']['prompt_tokens']
+            output_tokens = last_response_js['usage']['completion_tokens']
+            return input_tokens, output_tokens
 
-        Args:
-            responses (List[bytes]): List of http response body, for stream output,
-                there are multiple responses, for general only one.
-            kwargs: (Any): The command line --parameter content.
-        Returns:
-            Tuple: Return number of prompt token and number of completion tokens.
-        """
-        full_response_content = ''
+        # no usage information in the response, parse the response to get the tokens
         delta_contents = {}
-        input_tokens = None
-        output_tokens = None
         for response in responses:
             js = json.loads(response)
-            if js['object'] == 'chat.completion':
-                for choice in js['choices']:
-                    delta_contents[choice['index']] = [choice['message']['content']]
-                input_tokens = js['usage']['prompt_tokens']
-                output_tokens = js['usage']['completion_tokens']
-            elif js['object'] == 'text_completion':
-                for choice in js['choices']:
-                    delta_contents[choice['index']] = [choice['text']]
-                input_tokens = js['usage']['prompt_tokens']
-                output_tokens = js['usage']['completion_tokens']
-            elif js['object'] == 'chat.completion.chunk':
-                if 'choices' in js:
-                    for choice in js['choices']:
-                        if 'delta' in choice and 'index' in choice:
-                            delta = choice['delta']
-                            idx = choice['index']
-                            if 'content' in delta:
-                                delta_content = delta['content']
-                                if idx in delta_contents:
-                                    delta_contents[idx].append(delta_content)
-                                else:
-                                    delta_contents[idx] = [delta_content]
-                # usage in chunk: {"id":"","object":"chat.completion.chunk","created":1718269986,"model":"llama3",
-                # "choices":[],"usage":{"prompt_tokens":32,"total_tokens":384,"completion_tokens":352}}
-                if 'usage' in js and js['usage']:
-                    input_tokens = js['usage']['prompt_tokens']
-                    output_tokens = js['usage']['completion_tokens']
-        if (input_tokens is None and output_tokens is None and self.tokenizer is not None):
-            input_tokens = 0
-            output_tokens = 0
+            if 'object' in js:
+                self.__process_response_object(js, delta_contents)
+            else:
+                self.__process_no_object(js, delta_contents)
+
+        input_tokens, output_tokens = self.__calculate_tokens_from_content(request, delta_contents)
+        return input_tokens, output_tokens
+
+    def __process_response_object(self, js, delta_contents):
+        if js['object'] == 'chat.completion':
+            for choice in js['choices']:
+                delta_contents[choice['index']] = [choice['message']['content']]
+        elif js['object'] == 'text_completion':
+            for choice in js['choices']:
+                delta_contents[choice['index']] = [choice['text']]
+        elif js['object'] == 'chat.completion.chunk':
+            for choice in js.get('choices', []):
+                if 'delta' in choice and 'index' in choice:
+                    delta = choice['delta']
+                    idx = choice['index']
+                    if 'content' in delta:
+                        delta_content = delta['content']
+                        delta_contents.setdefault(idx, []).append(delta_content)
+
+    def __process_no_object(self, js, delta_contents):
+        #  assume the response is a single choice
+        for choice in js['choices']:
+            if 'delta' in choice:
+                delta = choice['delta']
+                idx = choice['index']
+                if 'content' in delta:
+                    delta_content = delta['content']
+                    delta_contents.setdefault(idx, []).append(delta_content)
+            else:
+                delta_contents[choice['index']] = [choice['message']['content']]
+
+    def __calculate_tokens_from_content(self, request, delta_contents):
+        input_tokens = output_tokens = 0
+        if self.tokenizer is not None:
             for idx, choice_contents in delta_contents.items():
-                full_response_content = ''.join([m for m in choice_contents])
+                full_response_content = ''.join(choice_contents)
                 input_tokens += len(self.tokenizer.encode(request['messages'][0]['content']))
                 output_tokens += len(self.tokenizer.encode(full_response_content))
-        elif input_tokens is None and output_tokens is None:  # no usage info get.
-            input_tokens = 0
-            output_tokens = 0
+        else:
             logger.warning('No usage information found. Please specify `--tokenizer-path` to generate usage details.')
-
         return input_tokens, output_tokens

evalscope/perf/plugin/datasets/flickr8k.py

@@ -1,6 +1,5 @@
 import base64
 from io import BytesIO
-from modelscope.msdatasets import MsDataset
 from PIL import Image
 from typing import Any, Dict, Iterator, List
 
@@ -26,6 +25,7 @@ class FlickrDatasetPlugin(DatasetPluginBase):
         super().__init__(query_parameters)
 
     def build_messages(self) -> Iterator[List[Dict]]:
+        from modelscope.msdatasets import MsDataset
         dataset = MsDataset.load('clip-benchmark/wds_flickr8k', split='test')
 
         for item in dataset:

evalscope/perf/plugin/datasets/longalpaca.py

@@ -1,4 +1,3 @@
-from modelscope import MsDataset
 from typing import Any, Dict, Iterator, List
 
 from evalscope.perf.arguments import Arguments
@@ -17,6 +16,7 @@ class LongAlpacaDatasetPlugin(DatasetPluginBase):
 
     def build_messages(self) -> Iterator[List[Dict]]:
         if not self.query_parameters.dataset_path:
+            from modelscope import MsDataset
             ds = MsDataset.load('AI-ModelScope/LongAlpaca-12k', subset_name='default', split='train')
         else:
             ds = self.dataset_json_list(self.query_parameters.dataset_path)

evalscope/perf/plugin/registry.py

@@ -1,4 +1,4 @@
-from typing import Any, List, Type
+from typing import Any, List, Type, Union
 
 
 class PluginRegistry:
@@ -20,7 +20,7 @@ class PluginRegistry:
         return self.get_class(name)
 
 
-def register_dataset(name: str | List[str]):
+def register_dataset(name: Union[str, List[str]]):
 
     def class_decorator(cls: Type):
         if isinstance(name, str):
@@ -35,7 +35,7 @@ def register_dataset(name: str | List[str]):
     return class_decorator
 
 
-def register_api(name: str | List[str]):
+def register_api(name: Union[str, List[str]]):
 
     def class_decorator(cls: Type):
         if isinstance(name, str):

evalscope/perf/utils/benchmark_util.py

@@ -116,19 +116,19 @@ class BenchmarkMetrics:
 
     def create_message(self, default_ndigits=3):
         message = {
-            'Time taken for tests (senconds)': round(self.total_time, default_ndigits),
+            'Time taken for tests (s)': round(self.total_time, default_ndigits),
             'Number of concurrency': self.concurrency,
             'Total requests': int(self.n_total_queries),
             'Succeed requests': self.n_succeed_queries,
             'Failed requests': self.n_failed_queries,
+            'Throughput(average tokens/s)': round(self.avg_token_per_seconds, default_ndigits),
             'Average QPS': round(self.qps, default_ndigits),
             'Average latency (s)': round(self.avg_latency, default_ndigits),
             'Average time to first token (s)': round(self.avg_first_chunk_latency, default_ndigits),
             'Average time per output token (s)': round(self.avg_time_per_token, 5),
-            'Average package latency (s)': round(self.avg_chunk_time, default_ndigits),
-            'Average package per request': round(self.n_avg_chunks, default_ndigits),
-            'Throughput(average output tokens per second)': round(self.avg_token_per_seconds, default_ndigits),
             'Average input tokens per request': round(self.avg_prompt_tokens, default_ndigits),
             'Average output tokens per request': round(self.avg_completion_tokens, default_ndigits),
+            'Average package latency (s)': round(self.avg_chunk_time, default_ndigits),
+            'Average package per request': round(self.n_avg_chunks, default_ndigits),
         }
         return message