evalscope 0.13.2__py3-none-any.whl → 0.14.0__py3-none-any.whl

evalscope/benchmarks/maritime_bench/maritime_bench_adapter.py

@@ -0,0 +1,79 @@
+from typing import Any
+
+from evalscope.benchmarks import Benchmark, DataAdapter
+from evalscope.constants import EvalType, OutputType
+from evalscope.metrics import exact_match
+from evalscope.utils.utils import ResponseParser
+
+SUBSET_LIST = ['default']
+
+
+@Benchmark.register(
+    name='maritime_bench',
+    pretty_name='MaritimeBench',
+    dataset_id='HiDolphin/MaritimeBench',
+    model_adapter=OutputType.GENERATION,
+    output_types=[OutputType.MULTIPLE_CHOICE, OutputType.GENERATION],
+    subset_list=SUBSET_LIST,
+    metric_list=['AverageAccuracy'],
+    eval_split='test',
+    prompt_template=
+    '题目来自于{subset_name}请回答单选题。要求只输出选项，不输出解释，将选项放在<>里，直接输出答案。示例：\n\n题目：在船舶主推进动力装置中，传动轴系在运转中承受以下复杂的应力和负荷，但不包括______。\n选项：\nA. 电磁力\nB. 压拉应力\nC. 弯曲应力\nD. 扭应力\n答：<A> 当前题目\n {query}',  # noqa: E501
+)
+class MaritimeBenchAdapter(DataAdapter):
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+        self.choices = ['A', 'B', 'C', 'D']
+
+    def gen_prompt(self, input_d: dict, subset_name: str, few_shot_list: list, **kwargs) -> Any:
+
+        prefix = ''
+        query = prefix + input_d['question'] + '\n'
+        available_choices = []
+        for option in self.choices:
+            if option in input_d and input_d[option]:
+                query += option + ':' + input_d[option] + '\n'
+                available_choices.append(option)
+
+        full_prompt = self.prompt_template.format(subset_name=subset_name, query=query)
+        return self.gen_prompt_data(full_prompt, choices=available_choices)
+
+    def get_gold_answer(self, input_d: dict) -> str:
+        """
+        Parse the raw input labels (gold).
+
+        Args:
+            input_d: input raw data. Depending on the dataset.
+
+        Returns:
+            The parsed input. e.g. gold answer ... Depending on the dataset.
+        """
+        return input_d['answer']
+
+    def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = EvalType.CHECKPOINT) -> str:
+        """
+        Parse the raw model prediction (pred).
+
+        Args:
+            pred: model prediction. Depending on the model.
+
+        Returns:
+            The parsed prediction. e.g. model answer... Depending on the model.
+        """
+
+        return ResponseParser.parse_bracketed_answer(result, options=self.choices)
+
+    def match(self, gold: Any, pred: Any) -> Any:
+        """
+        Match the gold answer with the predicted answer.
+
+        Args:
+            gold: The gold answer.
+            pred: The predicted answer.
+
+        Returns:
+            The result of the match.
+        """
+        return exact_match(gold=gold, pred=pred)

evalscope/benchmarks/mmlu/mmlu_adapter.py

@@ -145,7 +145,7 @@ SUBJECT_MAPPING = {
     train_split='train',
     eval_split='test',
     prompt_template=
-    'Answer the following multiple choice question about {subset_name}. There is only one correct answer. The last line of your response should be in the format "Answer: LETTER" (without quotes), where LETTER is one of A, B, C, D. \n{query}',
+    """Answer the following multiple choice question about {subset_name}. The last line of your response should be of the following format: 'Answer: $LETTER' (without quotes) where LETTER is one of ABCD. Think step by step before answering.\n\n{query}""",  # noqa: E501
 )
 class MMLUAdapter(DataAdapter):
 
@@ -224,9 +224,8 @@ class MMLUAdapter(DataAdapter):
 
         context: str = '\n'.join(few_shot_prompts) + '\n'
         context += self._generate_prompt(input_d=input_d, include_answer=False)
-        query = context.strip() + self._generate_prompt(input_d=input_d, include_answer=False)
 
-        full_prompt = self.prompt_template.format(subset_name=self._format_subject(subset_name), query=query)
+        full_prompt = self.prompt_template.format(subset_name=self._format_subject(subset_name), query=context.strip())
 
         return self.gen_prompt_data(full_prompt)
 
@@ -249,7 +248,7 @@ class MMLUAdapter(DataAdapter):
         if self.model_adapter == OutputType.MULTIPLE_CHOICE:
             return result
         else:
-            return ResponseParser.parse_first_option(result)
+            return ResponseParser.parse_first_option(result, options=self.choices)
 
     def match(self, gold: str, pred: str) -> float:
         return exact_match(gold=gold, pred=pred)
@@ -260,11 +259,10 @@ class MMLUAdapter(DataAdapter):
 
         example: str = input_d['input']
         for j in range(len(self.choices)):
-            example += '\n{}. {}'.format(self.choices[j], input_choices[j])
+            example += f'\n{self.choices[j]}) {input_choices[j]}'
 
-        example += '\nAnswer:'
         if include_answer:
-            example += ' {}\n\n'.format(input_d['target'])
+            example += f"\nAnswer: {input_d['target']}\n\n"
 
         return example
 

evalscope/benchmarks/mmlu_pro/mmlu_pro_adapter.py

@@ -92,7 +92,7 @@ class MMLUProAdapter(DataAdapter):
         if self.model_adapter == OutputType.MULTIPLE_CHOICE:
             return result
         else:
-            return ResponseParser.parse_first_option(result)
+            return ResponseParser.parse_first_option(result, options=self.choices)
 
     def match(self, gold: str, pred: str) -> float:
         """

evalscope/benchmarks/mmlu_redux/mmlu_redux_adapter.py

@@ -164,7 +164,7 @@ class MMLUReduxAdapter(DataAdapter):
         if self.model_adapter == OutputType.MULTIPLE_CHOICE:
             return result
         else:
-            return ResponseParser.parse_first_option(result)
+            return ResponseParser.parse_first_option(result, options=self.choices)
 
     def match(self, gold: str, pred: str) -> float:
         """

evalscope/benchmarks/musr/musr_adapter.py

@@ -62,7 +62,7 @@ class MuSRAdapter(DataAdapter):
         if self.model_adapter == OutputType.MULTIPLE_CHOICE:
             return result
         else:
-            return ResponseParser.parse_first_option(result)
+            return ResponseParser.parse_first_option(result, options=self.choices)
 
     def match(self, gold: str, pred: str) -> float:
         """

evalscope/collections/evaluator.py

@@ -65,7 +65,7 @@ class EvaluatorCollection:
         self.evaluators = self._initialize_evaluators()
 
     def load(self) -> tuple[list[DatasetEntry], str]:
-        dataset_name = os.path.basename(self.data_adapter.dataset_id).split('.')[0]
+        dataset_name = os.path.splitext(os.path.basename(self.data_adapter.dataset_id))[0]
         raw_dataset = self.data_adapter.load()
         # limit the dataset
         if self.task_cfg.limit:
@@ -174,6 +174,7 @@ class EvaluatorCollection:
         os.makedirs(os.path.dirname(report_file_path), exist_ok=True)
         with open(report_file_path, 'w', encoding='utf-8') as f:
             json.dump(report.to_dict(), f, ensure_ascii=False, indent=4)
+        return report
 
     def _filter_answer(self, pred_file_path):
         answer_dict = defaultdict(dict)
@@ -274,4 +275,5 @@ class EvaluatorCollection:
         answers = self.get_answers()
         reviews = self.get_reviews(answers)
         scores = self.get_scores(reviews)
-        self.get_report(scores)
+        report = self.get_report(scores)
+        return report

evalscope/config.py

@@ -212,7 +212,7 @@ def parse_task_config(task_cfg) -> TaskConfig:
         logger.info('Args: Task config is provided with CommandLine type.')
         task_cfg = TaskConfig.from_args(task_cfg)
     elif isinstance(task_cfg, str):
-        extension = task_cfg.split('.')[-1]
+        extension = os.path.splitext(task_cfg)[-1]
         logger.info(f'Args: Task config is provided with {extension} file type.')
         if extension in ['yaml', 'yml']:
             task_cfg = TaskConfig.from_yaml(task_cfg)

evalscope/perf/arguments.py

@@ -35,6 +35,7 @@ class Arguments:
     log_every_n_query: int = 10  # Log every N queries
     debug: bool = False  # Debug mode
     wandb_api_key: Optional[str] = None  # WandB API key for logging
+    swanlab_api_key: Optional[str] = None  # SwanLab API key for logging
     name: Optional[str] = None  # Name for the run
 
     # Output settings
@@ -46,6 +47,7 @@ class Arguments:
     prefix_length: int = 0  # Length of the prefix, only for random dataset
     prompt: Optional[str] = None  # The prompt text
     query_template: Optional[str] = None  # Template for the query
+    apply_chat_template: Optional[bool] = None  # Whether to apply chat template
 
     # Dataset settings
     dataset: str = 'openqa'  # Dataset type (default: 'line_by_line')
@@ -57,10 +59,10 @@ class Arguments:
     max_tokens: Optional[int] = 2048  # Maximum number of tokens in the response
     min_tokens: Optional[int] = None  # Minimum number of tokens in the response
     n_choices: Optional[int] = None  # Number of response choices
-    seed: Optional[int] = 42  # Random seed for reproducibility
+    seed: Optional[int] = 0  # Random seed for reproducibility
     stop: Optional[List[str]] = field(default_factory=list)  # Stop sequences for the response
     stop_token_ids: Optional[List[str]] = field(default_factory=list)  # Stop token IDs for the response
-    stream: Optional[bool] = False  # Whether to stream the response
+    stream: Optional[bool] = True  # Whether to stream the response
     temperature: float = 0.0  # Temperature setting for the response
     top_p: Optional[float] = None  # Top-p (nucleus) sampling setting for the response
     top_k: Optional[int] = None  # Top-k sampling setting for the response
@@ -76,12 +78,26 @@ class Arguments:
         return Arguments(**args_dict)
 
     def __post_init__(self):
+        # Set the default headers
         self.headers = self.headers or {}  # Default to empty dictionary
         if self.api_key:
             # Assuming the API key is used as a Bearer token
             self.headers['Authorization'] = f'Bearer {self.api_key}'
+
+        # Set the model ID based on the model name
         self.model_id = os.path.basename(self.model)
 
+        # Set the URL based on the dataset type
+        if self.api.startswith('local'):
+            if self.dataset.startswith('speed_benchmark'):
+                self.url = f'http://127.0.0.1:{self.port}/v1/completions'
+            else:
+                self.url = f'http://127.0.0.1:{self.port}/v1/chat/completions'
+
+        # Set the apply_chat_template flag based on the URL
+        if self.apply_chat_template is None:
+            self.apply_chat_template = self.url.strip('/').endswith('chat/completions')
+
     def __str__(self):
         return json.dumps(self.to_dict(), indent=4, default=str, ensure_ascii=False)
 
@@ -135,7 +151,8 @@ def add_argument(parser: argparse.ArgumentParser):
     parser.add_argument('--log-every-n-query', type=int, default=10, help='Logging every n query')
     parser.add_argument('--debug', action='store_true', default=False, help='Debug request send')
     parser.add_argument('--wandb-api-key', type=str, default=None, help='The wandb API key')
-    parser.add_argument('--name', type=str, help='The wandb db result name and result db name')
+    parser.add_argument('--swanlab-api-key', type=str, default=None, help='The swanlab API key')
+    parser.add_argument('--name', type=str, help='The wandb/swanlab db result name and result db name')
 
     # Prompt settings
     parser.add_argument('--max-prompt-length', type=int, default=sys.maxsize, help='Maximum input prompt length')
@@ -143,6 +160,8 @@ def add_argument(parser: argparse.ArgumentParser):
     parser.add_argument('--prefix-length', type=int, default=0, help='The prefix length')
     parser.add_argument('--prompt', type=str, required=False, default=None, help='Specified the request prompt')
     parser.add_argument('--query-template', type=str, default=None, help='Specify the query template')
+    parser.add_argument(
+        '--apply-chat-template', type=argparse.BooleanOptionalAction, default=None, help='Apply chat template to the prompt')  # noqa: E501
 
     # Output settings
     parser.add_argument('--outputs-dir', help='Outputs dir.', default='outputs')
@@ -159,10 +178,10 @@ def add_argument(parser: argparse.ArgumentParser):
     parser.add_argument(
         '--min-tokens', type=int, help='The minimum number of tokens that can be generated', default=None)
     parser.add_argument('--n-choices', type=int, help='How many completion choices to generate', default=None)
-    parser.add_argument('--seed', type=int, help='The random seed', default=42)
+    parser.add_argument('--seed', type=int, help='The random seed', default=0)
     parser.add_argument('--stop', nargs='*', help='The stop tokens', default=None)
     parser.add_argument('--stop-token-ids', nargs='*', help='Set the stop token IDs', default=None)
-    parser.add_argument('--stream', action='store_true', help='Stream output with SSE', default=False)
+    parser.add_argument('--stream', action=argparse.BooleanOptionalAction, help='Stream output with SSE', default=True)
     parser.add_argument('--temperature', type=float, help='The sample temperature', default=0.0)
     parser.add_argument('--top-p', type=float, help='Sampling top p', default=None)
     parser.add_argument('--top-k', type=int, help='Sampling top k', default=None)

evalscope/perf/benchmark.py

@@ -18,6 +18,7 @@ from evalscope.perf.utils.benchmark_util import BenchmarkData, BenchmarkMetrics
 from evalscope.perf.utils.db_util import create_result_table, get_result_db_path, insert_benchmark_data, summary_result
 from evalscope.perf.utils.handler import add_signal_handlers, exception_handler
 from evalscope.perf.utils.local_server import start_app
+from evalscope.perf.utils.log_utils import init_swanlab, init_wandb
 from evalscope.utils.logger import get_logger
 
 logger = get_logger()
@@ -56,7 +57,7 @@ async def get_requests(args: Arguments) -> AsyncGenerator[dict, None]:
 
     if args.prompt:
         prompt = load_prompt(args.prompt)
-        messages = [{'role': 'user', 'content': prompt}]
+        messages = [{'role': 'user', 'content': prompt}] if args.apply_chat_template else prompt
         generator = generate_requests_from_prompt(messages)
     elif args.dataset:
         generator = generate_requests_from_dataset()
@@ -81,6 +82,7 @@ async def send_request(
         client = AioHttpClient(args)
         async with client:
             benchmark_data = BenchmarkData(request=request)
+            benchmark_data.start_time = time.perf_counter()
             collected_messages = []
             try:
                 async for is_error, state_code, response_data in client.post(request):
@@ -106,24 +108,18 @@ async def send_request(
 
 
 @exception_handler
-async def statistic_benchmark_metric_worker(benchmark_data_queue: asyncio.Queue, args: Arguments):
+async def statistic_benchmark_metric(benchmark_data_queue: asyncio.Queue, args: Arguments):
     metrics = BenchmarkMetrics(concurrency=args.parallel)
 
     api_plugin_class = ApiRegistry(args.api)
     api_plugin = api_plugin_class(args.tokenizer_path)
 
     result_db_path = get_result_db_path(args)
-    # Initialize wandb
-    if args.wandb_api_key:
-        import datetime
-        import wandb
-        os.environ['WANDB_SILENT'] = 'true'
-        os.environ['WANDB_DIR'] = args.outputs_dir
 
-        wandb.login(key=args.wandb_api_key)
-        current_time = datetime.datetime.now().strftime('%Y%m%d_%H%M%S')
-        name = args.name if args.name else f'{args.model_id}_{current_time}'
-        wandb.init(project='perf_benchmark', name=name, config=args.to_dict())
+    if args.wandb_api_key:
+        init_wandb(args)
+    if args.swanlab_api_key:
+        init_swanlab(args)
 
     collected_benchmark_data = []
 
@@ -146,9 +142,13 @@ async def statistic_benchmark_metric_worker(benchmark_data_queue: asyncio.Queue,
             # Create a message with the updated metrics
             message = metrics.create_message()
 
-            # Log the message to wandb if the api key is provided
+            # Log the message to wandb\swanlab if the api key is provided
             if args.wandb_api_key:
+                import wandb
                 wandb.log(message)
+            if args.swanlab_api_key:
+                import swanlab
+                swanlab.log(message)
 
             # Log the message to the logger every n queries
             if int(metrics.n_total_queries) % args.log_every_n_query == 0:
@@ -169,17 +169,12 @@ async def statistic_benchmark_metric_worker(benchmark_data_queue: asyncio.Queue,
 
 
 @exception_handler
-async def start_server(args: Arguments) -> bool:
+async def connect_test(args: Arguments) -> bool:
     if args.api.startswith('local'):
         #  start local server
         server = threading.Thread(target=start_app, args=(copy.deepcopy(args), ), daemon=True)
         server.start()
 
-        if args.dataset.startswith('speed_benchmark'):
-            args.url = f'http://127.0.0.1:{args.port}/v1/completions'
-        else:
-            args.url = f'http://127.0.0.1:{args.port}/v1/chat/completions'
-
     if (not args.no_test_connection) and (not await test_connection(args)):
         raise TimeoutError('Test connection failed')
 
@@ -192,31 +187,22 @@ async def benchmark(args: Arguments) -> None:
 
     # init queue
     benchmark_data_queue = asyncio.Queue()
-
     # reset event
     data_process_completed_event.clear()
-
+    # test connection
+    await connect_test(args)
+    # start statistic benchmark metric
+    statistic_benchmark_metric_task = asyncio.create_task(statistic_benchmark_metric(benchmark_data_queue, args))
+    # start send request
     semaphore = asyncio.Semaphore(args.parallel)
+    send_request_tasks: List[asyncio.Task] = []
+    async for request in get_requests(args):
+        task = asyncio.create_task(send_request(semaphore, request, benchmark_data_queue, args))
+        send_request_tasks.append(task)
 
-    async def create_send_request_tasks():
-        tasks: List[asyncio.Task] = []
-        async for request in get_requests(args):
-            task = asyncio.create_task(send_request(semaphore, request, benchmark_data_queue, args))
-            tasks.append(task)
-        return tasks
-
-    async def run_tasks():
-        await start_server(args)
-
-        statistic_benchmark_metric_task = asyncio.create_task(
-            statistic_benchmark_metric_worker(benchmark_data_queue, args))
-        send_request_tasks = await create_send_request_tasks()
-
-        await asyncio.gather(*send_request_tasks, return_exceptions=True)
-        await benchmark_data_queue.join()
-        data_process_completed_event.set()
-
-        metrics, result_db_path = await statistic_benchmark_metric_task
-        summary_result(args, metrics, result_db_path)
+    await asyncio.gather(*send_request_tasks, return_exceptions=True)
+    await benchmark_data_queue.join()
+    data_process_completed_event.set()
 
-    await run_tasks()
+    metrics, result_db_path = await statistic_benchmark_metric_task
+    summary_result(args, metrics, result_db_path)

evalscope/perf/http_client.py

@@ -24,7 +24,6 @@ class AioHttpClient:
         self.connect_timeout = args.connect_timeout
         self.client = aiohttp.ClientSession(
             timeout=aiohttp.ClientTimeout(connect=self.connect_timeout, sock_read=self.read_timeout),
-            connector=aiohttp.TCPConnector(limit=1),
             trace_configs=[self._create_trace_config()] if args.debug else [])
 
     def _create_trace_config(self):
@@ -144,7 +143,7 @@ async def test_connection(args: Arguments) -> bool:
     async def attempt_connection():
         client = AioHttpClient(args)
         async with client:
-            if 'chat/completions' in args.url:
+            if args.apply_chat_template:
                 request = {
                     'messages': [{
                         'role': 'user',
@@ -164,7 +163,7 @@ async def test_connection(args: Arguments) -> bool:
             is_error, state_code, response_data = await asyncio.wait_for(
                 attempt_connection(), timeout=args.connect_timeout)
             if not is_error:
-                logger.info('Connection successful.')
+                logger.info('Test connection successful.')
                 return True
             logger.warning(f'Retrying...  <{state_code}> {response_data}')
         except Exception as e:

evalscope/perf/plugin/api/custom_api.py

@@ -24,7 +24,7 @@ class CustomPlugin(ApiPluginBase):
         """
         super().__init__(model_path=mode_path)
         if mode_path is not None:
-            from transformers import AutoTokenizer
+            from modelscope import AutoTokenizer
             self.tokenizer = AutoTokenizer.from_pretrained(mode_path)
         else:
             self.tokenizer = None

evalscope/perf/plugin/api/openai_api.py

@@ -24,7 +24,7 @@ class OpenaiPlugin(ApiPluginBase):
         """
         super().__init__(model_path=mode_path)
         if mode_path is not None:
-            from transformers import AutoTokenizer
+            from modelscope import AutoTokenizer
             self.tokenizer = AutoTokenizer.from_pretrained(mode_path)
         else:
             self.tokenizer = None
@@ -70,7 +70,7 @@ class OpenaiPlugin(ApiPluginBase):
     def __compose_query_from_parameter(self, payload: Dict, param: Arguments):
         payload['model'] = param.model
         if param.max_tokens is not None:
-            payload['max_completion_tokens'] = param.max_tokens
+            payload['max_tokens'] = param.max_tokens
         if param.min_tokens is not None:
             payload['min_tokens'] = param.min_tokens
         if param.frequency_penalty is not None:

evalscope/perf/plugin/datasets/custom.py

@@ -18,4 +18,7 @@ class CustomDatasetPlugin(DatasetPluginBase):
             prompt = item.strip()
             if len(prompt) > self.query_parameters.min_prompt_length and len(
                     prompt) < self.query_parameters.max_prompt_length:
-                yield [{'role': 'user', 'content': prompt}]
+                if self.query_parameters.apply_chat_template:
+                    yield [{'role': 'user', 'content': prompt}]
+                else:
+                    yield prompt

evalscope/perf/plugin/datasets/line_by_line.py

@@ -19,4 +19,7 @@ class LineByLineDatasetPlugin(DatasetPluginBase):
             prompt = item.strip()
             if len(prompt) > self.query_parameters.min_prompt_length and len(
                     prompt) < self.query_parameters.max_prompt_length:
-                yield [{'role': 'user', 'content': prompt}]
+                if self.query_parameters.apply_chat_template:
+                    yield [{'role': 'user', 'content': prompt}]
+                else:
+                    yield prompt

evalscope/perf/plugin/datasets/longalpaca.py

@@ -24,4 +24,7 @@ class LongAlpacaDatasetPlugin(DatasetPluginBase):
             prompt = item['instruction'].strip()
             if len(prompt) > self.query_parameters.min_prompt_length and len(
                     prompt) < self.query_parameters.max_prompt_length:
-                yield [{'role': 'user', 'content': prompt}]
+                if self.query_parameters.apply_chat_template:
+                    yield [{'role': 'user', 'content': prompt}]
+                else:
+                    yield prompt

evalscope/perf/plugin/datasets/openqa.py

@@ -29,4 +29,7 @@ class OpenqaDatasetPlugin(DatasetPluginBase):
             prompt = item['question'].strip()
             if (len(prompt) > self.query_parameters.min_prompt_length
                     and len(prompt) < self.query_parameters.max_prompt_length):
-                yield [{'role': 'user', 'content': prompt}]
+                if self.query_parameters.apply_chat_template:
+                    yield [{'role': 'user', 'content': prompt}]
+                else:
+                    yield prompt

evalscope/perf/plugin/datasets/random_dataset.py

@@ -23,8 +23,12 @@ class RandomDatasetPlugin(DatasetPluginBase):
         self.number = self.query_parameters.number or 1
 
     def build_messages(self) -> Iterator[List[Dict]]:
-        min_prompt_length = self.query_parameters.min_prompt_length - self.template_len
-        max_prompt_length = self.query_parameters.max_prompt_length - self.template_len + 1
+        if self.query_parameters.apply_chat_template:
+            min_prompt_length = self.query_parameters.min_prompt_length - self.template_len
+            max_prompt_length = self.query_parameters.max_prompt_length - self.template_len + 1
+        else:
+            min_prompt_length = self.query_parameters.min_prompt_length
+            max_prompt_length = self.query_parameters.max_prompt_length + 1
 
         assert min_prompt_length >= 0, f'min_prompt_length should be greater than or equal to the template length {self.template_len}.'  # noqa: E501
         assert max_prompt_length >= min_prompt_length, 'max_prompt_length should be greater than or equal to min_prompt_length.'  # noqa: E501
@@ -34,10 +38,13 @@ class RandomDatasetPlugin(DatasetPluginBase):
         offsets = np.random.randint(0, self.tokenizer.vocab_size, size=self.number)
 
         for i in range(self.number):
-            prompt_ids = (offsets[i] + i + np.arange(input_lens[i])) % self.tokenizer.vocab_size
-            prompt = self.tokenizer.decode(
-                self.prefix_ids + prompt_ids.tolist(), skip_special_tokens=False, clean_up_tokenization_spaces=False)
-            yield [{'role': 'user', 'content': prompt}]
+            prompt_ids = ((offsets[i] + i + np.arange(input_lens[i])) % self.tokenizer.vocab_size).tolist()
+            prompt = self.tokenizer.decode(self.prefix_ids + prompt_ids)
+
+            if self.query_parameters.apply_chat_template:
+                yield [{'role': 'user', 'content': prompt}]
+            else:
+                yield prompt
 
     def get_random_inputs(self, length: int) -> List[int]:
         if length <= 0:

evalscope/perf/utils/benchmark_util.py

@@ -11,7 +11,7 @@ logger = get_logger()
 @dataclass
 class BenchmarkData:
     request: Any = None
-    start_time: float = field(default_factory=time.perf_counter)
+    start_time: float = 0.0
     completed_time: float = 0.0
     chunk_times: List[float] = field(default_factory=list)
     success: bool = False
@@ -73,7 +73,9 @@ class BenchmarkMetrics:
     avg_chunk_time: float = -1
     avg_prompt_tokens: float = -1
     avg_completion_tokens: float = -1
-    avg_token_per_seconds: float = -1
+    avg_input_token_per_seconds: float = -1
+    avg_output_token_per_seconds: float = -1
+    avg_total_token_per_seconds: float = -1
     avg_time_per_token: float = -1
     qps: float = -1
 
@@ -111,22 +113,26 @@ class BenchmarkMetrics:
             self.avg_chunk_time = self.total_chunks_time / self.n_total_chunks
             self.avg_prompt_tokens = self.n_total_prompt_tokens / self.n_succeed_queries
             self.avg_completion_tokens = self.n_total_completion_tokens / self.n_succeed_queries
-            self.avg_token_per_seconds = self.n_total_completion_tokens / self.total_time
+            self.avg_input_token_per_seconds = self.n_total_prompt_tokens / self.total_first_chunk_latency
+            self.avg_output_token_per_seconds = self.n_total_completion_tokens / self.total_time
+            self.avg_total_token_per_seconds = (self.n_total_prompt_tokens
+                                                + self.n_total_completion_tokens) / self.total_time
             self.avg_time_per_token = self.n_time_per_output_token / self.n_succeed_queries
             self.qps = self.n_succeed_queries / self.total_time
         except ZeroDivisionError as e:
             logger.exception(e)
             return
 
-    def create_message(self, default_ndigits=3):
+    def create_message(self, default_ndigits=4):
         message = {
             'Time taken for tests (s)': round(self.total_time, default_ndigits),
             'Number of concurrency': self.concurrency,
             'Total requests': int(self.n_total_queries),
             'Succeed requests': self.n_succeed_queries,
             'Failed requests': self.n_failed_queries,
-            'Throughput(average tokens/s)': round(self.avg_token_per_seconds, default_ndigits),
-            'Average QPS': round(self.qps, default_ndigits),
+            'Output token throughput (tok/s)': round(self.avg_output_token_per_seconds, default_ndigits),
+            'Total token throughput (tok/s)': round(self.avg_total_token_per_seconds, default_ndigits),
+            'Request throughput (req/s)': round(self.qps, default_ndigits),
             'Average latency (s)': round(self.avg_latency, default_ndigits),
             'Average time to first token (s)': round(self.avg_first_chunk_latency, default_ndigits),
             'Average time per output token (s)': round(self.avg_time_per_token, default_ndigits),

evalscope/perf/utils/db_util.py

@@ -175,7 +175,7 @@ def get_percentile_results(result_db_path: str) -> Dict[str, List[float]]:
 
     metrics = {
         'TTFT (s)': [row[FIRST_CHUNK_LATENCY_INDEX] for row in rows],
-        'TPOT (s)':
+        'ITL (s)':
         inter_token_latencies_all,
         'Latency (s)': [row[LATENCY_INDEX] for row in rows],
         'Input tokens': [row[PROMPT_TOKENS_INDEX] for row in rows],

evalscope/perf/utils/log_utils.py

@@ -0,0 +1,41 @@
+import os
+
+from evalscope.perf.arguments import Arguments
+
+
+def init_wandb(args: Arguments) -> None:
+    """
+    Initialize WandB for logging.
+    """
+    # Initialize wandb if the api key is provided
+    import datetime
+    try:
+        import wandb
+    except ImportError:
+        raise RuntimeError('Cannot import wandb. Please install it with command: \n pip install wandb')
+    os.environ['WANDB_SILENT'] = 'true'
+    os.environ['WANDB_DIR'] = args.outputs_dir
+
+    wandb.login(key=args.wandb_api_key)
+    current_time = datetime.datetime.now().strftime('%Y%m%d_%H%M%S')
+    name = args.name if args.name else f'{args.model_id}_{current_time}'
+    wandb.init(project='perf_benchmark', name=name, config=args.to_dict())
+
+
+def init_swanlab(args: Arguments) -> None:
+    import datetime
+    try:
+        import swanlab
+    except ImportError:
+        raise RuntimeError('Cannot import swanlab. Please install it with command: \n pip install swanlab')
+    os.environ['SWANLAB_LOG_DIR'] = args.outputs_dir
+    if not args.swanlab_api_key == 'local':
+        swanlab.login(api_key=args.swanlab_api_key)
+    current_time = datetime.datetime.now().strftime('%Y%m%d_%H%M%S')
+    name = args.name if args.name else f'{args.model_id}_{current_time}'
+    swanlab.config.update({'framework': '📏evalscope'})
+    swanlab.init(
+        project='perf_benchmark',
+        name=name,
+        config=args.to_dict(),
+        mode='local' if args.swanlab_api_key == 'local' else None)