evalscope 0.6.1__py3-none-any.whl → 0.7.0__py3-none-any.whl

evalscope/metrics/rouge_metric.py

@@ -9,16 +9,24 @@ from tqdm import tqdm
 
 from evalscope.constants import MetricsConstant
 from evalscope.metrics.bundled_rouge_score import rouge_scorer
-from evalscope.preprocess.tokenizers.gpt2_tokenizer import DummyTokenizer
+
 from rouge_chinese import Rouge
 import jieba
 
+
+class DummyTokenizer:
+
+    def tokenize(self, text: str):
+        return text.split()
+
+
 HERE = Path(__file__).absolute().parent
 
 logger = logging.getLogger(__name__)
 
-scorer = rouge_scorer.RougeScorer(['rouge1', 'rouge2', 'rougeL'],
-                                  tokenizer=DummyTokenizer())
+scorer = rouge_scorer.RougeScorer(
+    ['rouge1', 'rouge2', 'rougeL'], tokenizer=DummyTokenizer()
+)
 zh_scorer = Rouge()
 
 
@@ -28,6 +36,7 @@ def is_contains_chinese(strs):
             return True
     return False
 
+
 def compute_rouge_score(predict_l, reference_l):
     assert len(predict_l) == len(reference_l)
     if len(predict_l) == 0:
@@ -43,10 +52,14 @@ def compute_rouge_score(predict_l, reference_l):
             result[rouge_key].append(one_sample[rouge_key])
     rlt = {}
     for rouge_key in MetricsConstant.ROUGE_KEYS:
-        rlt[rouge_key] = mean(result[rouge_key]) * 100 if rouge_key in result \
+        rlt[rouge_key] = (
+            mean(result[rouge_key]) * 100
+            if rouge_key in result
             else MetricsConstant.INVALID_VALUE
+        )
     return rlt
 
+
 def compute_rouge_score_one_sample_zh(predict, reference):
     result = dict()
     for p, r in zip(predict, reference):
@@ -63,9 +76,10 @@ def compute_rouge_score_one_sample_zh(predict, reference):
         result['rouge-l-r'] = score['rouge-l']['r']
         result['rouge-l-p'] = score['rouge-l']['p']
         result['rouge-l-f'] = score['rouge-l']['f']
-    
+
     return result
 
+
 def compute_rouge_score_one_sample(predict, reference):
     result = dict()
     for p, r in zip(predict, reference):
@@ -97,11 +111,9 @@ def _to_table(final_result) -> str:
             if not task:
                 continue
             elif task == 'total':
-                row.append(
-                    f'{final_result["total"]["rouge"][rouge_key] :0.2f}')
+                row.append(f'{final_result["total"]["rouge"][rouge_key] :0.2f}')
             else:
-                row.append(
-                    f'{final_result["tasks"][task]["rouge"][rouge_key] :0.2f}')
+                row.append(f'{final_result["tasks"][task]["rouge"][rouge_key] :0.2f}')
         table.append('\t'.join(row))
 
     return '\n'.join(table)
@@ -111,19 +123,22 @@ def run_rouge_eval(data_l, md_level=2, report_metric_key='rouge-l-f'):
     print(f"{'#' * md_level} Rouge Eval")
     for data in tqdm(data_l):
         data['rouge'] = compute_rouge_score_one_sample(
-            data['gen_tok_str'], data['reference_tok_str'])
+            data['gen_tok_str'], data['reference_tok_str']
+        )
     task_data_d = defaultdict(list)
     for data in data_l:
         for task in data['task_tags']:
             task_data_d[task].append(data)
 
     total_rouge = mean([data['rouge'][report_metric_key] for data in data_l])
-    print(f'[total], count: {len(data_l)}, {report_metric_key}: '
-          f'{total_rouge * 100:0.2f}%')
+    print(
+        f'[total], count: {len(data_l)}, {report_metric_key}: '
+        f'{total_rouge * 100:0.2f}%'
+    )
 
     for task, task_data in task_data_d.items():
-        task_rouge = mean(
-            [data['rouge'][report_metric_key] for data in task_data])
+        task_rouge = mean([data['rouge'][report_metric_key] for data in task_data])
         print(
             f'[{task}], count: {len(task_data_d[task])}, {report_metric_key}: '
-            f'{task_rouge * 100:0.2f}%')
+            f'{task_rouge * 100:0.2f}%'
+        )

evalscope/perf/arguments.py

@@ -0,0 +1,179 @@
+import argparse
+import sys
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional
+
+import json
+
+
+@dataclass
+class Arguments:
+    # Model and API
+    model: str  # Model identifier
+    attn_implementation: Optional[str] = None  # Attention implementaion, only for local inference
+    api: str = 'openai'  # API to be used (default: 'openai')
+    tokenizer_path: Optional[str] = None  # Path to the tokenizer
+
+    # Connection settings
+    url: str = 'http://127.0.0.1:8877/v1/chat/completions'  # URL for the API connection
+    headers: Dict[str, Any] = field(default_factory=dict)  # Custom headers
+    connect_timeout: int = 120  # Connection timeout in seconds
+    read_timeout: int = 120  # Read timeout in seconds
+    api_key: str = 'EMPTY'
+
+    # Performance and parallelism
+    number: Optional[int] = None  # Number of requests to be made
+    parallel: int = 1  # Number of parallel requests
+    rate: int = -1  # Rate limit for requests (default: -1, no limit)
+
+    # Logging and debugging
+    log_every_n_query: int = 10  # Log every N queries
+    debug: bool = False  # Debug mode
+    wandb_api_key: Optional[str] = None  # WandB API key for logging
+    name: Optional[str] = None  # Name for the run
+
+    # Prompt settings
+    max_prompt_length: int = sys.maxsize  # Maximum length of the prompt
+    min_prompt_length: int = 0  # Minimum length of the prompt
+    prompt: Optional[str] = None  # The prompt text
+    query_template: Optional[str] = None  # Template for the query
+
+    # Dataset settings
+    dataset: str = 'openqa'  # Dataset type (default: 'line_by_line')
+    dataset_path: Optional[str] = None  # Path to the dataset
+
+    # Response settings
+    frequency_penalty: Optional[float] = None  # Frequency penalty for the response
+    logprobs: Optional[bool] = None  # Whether to log probabilities
+    max_tokens: Optional[int] = 2048  # Maximum number of tokens in the response
+    min_tokens: Optional[int] = None  # Minimum number of tokens in the response
+    n_choices: Optional[int] = None  # Number of response choices
+    seed: Optional[int] = 42  # Random seed for reproducibility
+    stop: Optional[List[str]] = field(default_factory=list)  # Stop sequences for the response
+    stop_token_ids: Optional[List[str]] = field(default_factory=list)  # Stop token IDs for the response
+    stream: Optional[bool] = None  # Whether to stream the response
+    temperature: Optional[float] = None  # Temperature setting for the response
+    top_p: Optional[float] = None  # Top-p (nucleus) sampling setting for the response
+
+    @staticmethod
+    def from_args(args):
+
+        return Arguments(
+            model=args.model,
+            attn_implementation=args.attn_implementation,
+            url=args.url,
+            api_key=args.api_key,
+            connect_timeout=args.connect_timeout,
+            read_timeout=args.read_timeout,
+            number=args.number,
+            parallel=args.parallel,
+            rate=args.rate,
+            log_every_n_query=args.log_every_n_query,
+            headers=args.headers,
+            wandb_api_key=args.wandb_api_key,
+            name=args.name,
+            debug=args.debug,
+            tokenizer_path=args.tokenizer_path,
+            api=args.api,
+            max_prompt_length=args.max_prompt_length,
+            min_prompt_length=args.min_prompt_length,
+            prompt=args.prompt,
+            query_template=args.query_template,
+            dataset=args.dataset,
+            dataset_path=args.dataset_path,
+            frequency_penalty=args.frequency_penalty,
+            logprobs=args.logprobs,
+            max_tokens=args.max_tokens,
+            min_tokens=args.min_tokens,
+            n_choices=args.n_choices,
+            seed=args.seed,
+            stop=args.stop,
+            stop_token_ids=args.stop_token_ids,
+            stream=args.stream,
+            temperature=args.temperature,
+            top_p=args.top_p)
+
+    def __post_init__(self):
+        self.headers = self.headers or {}  # Default to empty dictionary
+        if self.api_key:
+            # Assuming the API key is used as a Bearer token
+            self.headers['Authorization'] = f'Bearer {self.api_key}'
+
+    def __str__(self):
+        return json.dumps(self.to_dict(), indent=4, default=str, ensure_ascii=False)
+
+    def to_dict(self) -> Dict[str, Any]:
+        return self.__dict__
+
+
+class ParseKVAction(argparse.Action):
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        if not values:
+            setattr(namespace, self.dest, {})
+        else:
+            try:
+                kv_dict = dict(kv.split('=') for kv in values)
+                setattr(namespace, self.dest, kv_dict)
+            except ValueError as e:
+                parser.error(f'Error parsing key-value pairs: {e}')
+
+
+def add_argument(parser: argparse.ArgumentParser):
+    # yapf: disable
+    # Model and API
+    parser.add_argument('--model', type=str, required=True, help='The test model name.')
+    parser.add_argument('--attn-implementation', required=False, default=None, help='Attention implementaion')
+    parser.add_argument('--api', type=str, default='openai', help='Specify the service API')
+    parser.add_argument(
+        '--tokenizer-path', type=str, required=False, default=None, help='Specify the tokenizer weight path')
+
+    # Connection settings
+    parser.add_argument('--url', type=str, default='http://127.0.0.1:8877/v1/chat/completions')
+    parser.add_argument('--headers', nargs='+', dest='headers', action=ParseKVAction, help='Extra HTTP headers')
+    parser.add_argument('--api-key', type=str, required=False, default='EMPTY', help='The API key for authentication')
+    parser.add_argument('--connect-timeout', type=int, default=120, help='The network connection timeout')
+    parser.add_argument('--read-timeout', type=int, default=120, help='The network read timeout')
+
+    # Performance and parallelism
+    parser.add_argument('-n', '--number', type=int, default=None, help='How many requests to be made')
+    parser.add_argument('--parallel', type=int, default=1, help='Set number of concurrency requests, default 1')
+    parser.add_argument('--rate', type=int, default=-1, help='Number of requests per second. default None')
+
+    # Logging and debugging
+    parser.add_argument('--log-every-n-query', type=int, default=10, help='Logging every n query')
+    parser.add_argument('--debug', action='store_true', default=False, help='Debug request send')
+    parser.add_argument('--wandb-api-key', type=str, default=None, help='The wandb API key')
+    parser.add_argument('--name', type=str, help='The wandb db result name and result db name')
+
+    # Prompt settings
+    parser.add_argument('--max-prompt-length', type=int, default=sys.maxsize, help='Maximum input prompt length')
+    parser.add_argument('--min-prompt-length', type=int, default=0, help='Minimum input prompt length')
+    parser.add_argument('--prompt', type=str, required=False, default=None, help='Specified the request prompt')
+    parser.add_argument('--query-template', type=str, default=None, help='Specify the query template')
+
+    # Dataset settings
+    parser.add_argument('--dataset', type=str, default='openqa', help='Specify the dataset')
+    parser.add_argument('--dataset-path', type=str, required=False, help='Path to the dataset file')
+
+    # Response settings
+    parser.add_argument('--frequency-penalty', type=float, help='The frequency_penalty value', default=None)
+    parser.add_argument('--logprobs', action='store_true', help='The logprobs', default=None)
+    parser.add_argument(
+        '--max-tokens', type=int, help='The maximum number of tokens that can be generated', default=2048)
+    parser.add_argument(
+        '--min-tokens', type=int, help='The minimum number of tokens that can be generated', default=None)
+    parser.add_argument('--n-choices', type=int, help='How many completion choices to generate', default=None)
+    parser.add_argument('--seed', type=int, help='The random seed', default=42)
+    parser.add_argument('--stop', nargs='*', help='The stop tokens', default=None)
+    parser.add_argument('--stop-token-ids', nargs='*', help='Set the stop token IDs', default=None)
+    parser.add_argument('--stream', action='store_true', help='Stream output with SSE', default=None)
+    parser.add_argument('--temperature', type=float, help='The sample temperature', default=None)
+    parser.add_argument('--top-p', type=float, help='Sampling top p', default=None)
+    # yapf: enable
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description='Benchmark LLM service performance.')
+    add_argument(parser)
+    return parser.parse_args()

evalscope/perf/benchmark.py

@@ -0,0 +1,245 @@
+import asyncio
+import copy
+import os
+import platform
+import sqlite3
+import threading
+import time
+from http import HTTPStatus
+from typing import List
+
+import json
+import numpy as np
+from tqdm import tqdm
+
+from evalscope.perf.arguments import Arguments
+from evalscope.perf.http_client import AioHttpClient, test_connection
+from evalscope.perf.plugin.registry import ApiRegistry, DatasetRegistry
+from evalscope.perf.utils.benchmark_util import BenchmarkData, BenchmarkMetrics
+from evalscope.perf.utils.db_util import create_result_table, get_result_db_path, insert_benchmark_data, summary_result
+from evalscope.perf.utils.handler import add_signal_handlers, exception_handler
+from evalscope.perf.utils.local_server import start_app
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+query_send_completed_event = asyncio.Event()
+data_process_completed_event = asyncio.Event()
+
+
+@exception_handler
+async def dispatch_requests_worker(request_queue: asyncio.Queue, args: Arguments):
+    query_generator_class = ApiRegistry(args.api)
+    query_generator = query_generator_class(args.tokenizer_path)
+
+    def load_prompt(prompt_path_or_text):
+        """Load the prompt from a file or directly from the input text."""
+        if prompt_path_or_text.startswith('@'):
+            with open(prompt_path_or_text[1:], 'r', encoding='utf-8') as file:
+                return file.read()
+        return prompt_path_or_text
+
+    async def dispatch_request(request):
+        """Dispatch a single request with optional rate limiting."""
+        await request_queue.put(request)
+        if args.rate != -1:
+            interval = np.random.exponential(1.0 / args.rate)
+            await asyncio.sleep(interval)
+
+    async def dispatch_requests_from_prompt(messages):
+        """Generate and dispatch requests based on the given prompt."""
+        request = query_generator.build_request(messages, args)
+        if args.number is None:
+            await dispatch_request(request)
+            return 1
+        for _ in range(args.number):
+            await dispatch_request(request)
+        return args.number
+
+    async def dispatch_requests_from_dataset():
+        """Generate and dispatch requests based on the dataset."""
+        total_query_count = 0
+        message_generator_class = DatasetRegistry(args.dataset)
+        message_generator = message_generator_class(args)
+
+        for messages in message_generator:
+            request = query_generator.build_request(messages, args)
+            if request is None:
+                continue
+            await dispatch_request(request)
+            total_query_count += 1
+            if args.number and total_query_count >= args.number:
+                break
+
+        return total_query_count
+
+    # Load prompt or dataset and dispatch requests accordingly
+    if args.prompt:
+        prompt = load_prompt(args.prompt)
+        messages = [{'role': 'user', 'content': prompt}]
+        total_queries = await dispatch_requests_from_prompt(messages)
+    elif args.dataset:
+        total_queries = await dispatch_requests_from_dataset()
+    else:
+        raise Exception('Either prompt or dataset is required!')
+
+    return total_queries
+
+
+@exception_handler
+async def send_requests_worker(
+    task_id,
+    request_queue: asyncio.Queue,
+    benchmark_data_queue: asyncio.Queue,
+    args: Arguments,
+):
+    client = AioHttpClient(args)
+    async with client:
+        while not (query_send_completed_event.is_set() and request_queue.empty()):
+            try:
+                # Attempt to get a request from the queue with a timeout
+                request = await asyncio.wait_for(request_queue.get(), timeout=0.0001)
+                request_queue.task_done()
+            except asyncio.TimeoutError:
+                # If timeout, continue to the next iteration
+                continue
+
+            # Initialize benchmark data for the current request
+            benchmark_data = BenchmarkData(request=request)
+            collected_messages = []
+            try:
+                # Send the request and process the response
+                async for is_error, state_code, response_data in client.post(request):
+                    if is_error or state_code != HTTPStatus.OK:
+                        logger.error(f'Request: {request} failed, state_code: {state_code}, data: {response_data}')
+                        benchmark_data.success = False
+                        break
+                    if response_data:
+                        collected_messages.append(response_data)
+                        benchmark_data.chunk_times.append(time.perf_counter())
+                        benchmark_data.success = True
+                        benchmark_data.update_gpu_usage()
+            except Exception as e:
+                if response_data:
+                    collected_messages.append(response_data)
+                benchmark_data.success = False
+                logger.exception(e)
+                logger.error(f'Request query: {request} exception')
+            finally:
+                # Record completion time and collected messages
+                benchmark_data.completed_time = time.perf_counter()
+                benchmark_data.response_messages = collected_messages
+                await benchmark_data_queue.put(benchmark_data)
+
+
+@exception_handler
+async def statistic_benchmark_metric_worker(benchmark_data_queue: asyncio.Queue, args: Arguments):
+    metrics = BenchmarkMetrics(concurrency=args.parallel)
+
+    api_plugin_class = ApiRegistry(args.api)
+    api_plugin = api_plugin_class(args.tokenizer_path)
+
+    result_db_path = get_result_db_path(args.name, args.model)
+    # Initialize wandb
+    if args.wandb_api_key:
+        import wandb
+        import datetime
+        os.environ['WANDB_SILENT'] = 'true'
+        os.environ['WANDB_DIR'] = './outputs'
+
+        wandb.login(key=args.wandb_api_key)
+        current_time = datetime.datetime.now().strftime('%Y%m%d_%H%M%S')
+        name = args.name if args.name else f'{args.model}_{current_time}'
+        wandb.init(project='perf_benchmark', name=name, config=args.to_dict())
+
+    with sqlite3.connect(result_db_path) as con:
+        cursor = con.cursor()
+        create_result_table(cursor)
+        with tqdm(desc='Processing') as pbar:
+            while not (data_process_completed_event.is_set() and benchmark_data_queue.empty()):
+                try:
+                    # Attempt to get benchmark data from the queue with a timeout
+                    benchmark_data = await asyncio.wait_for(benchmark_data_queue.get(), timeout=1)
+                    benchmark_data_queue.task_done()
+                except asyncio.TimeoutError:
+                    # If timeout, continue to the next iteration
+                    continue
+
+                # Update metrics based on the benchmark data
+                metrics.update_metrics(benchmark_data, api_plugin)
+
+                # Insert benchmark data into the database and commit the transaction
+                insert_benchmark_data(cursor, benchmark_data)
+                con.commit()
+
+                # Create a message with the updated metrics
+                message = metrics.create_message()
+
+                # Log the message to wandb if the api key is provided
+                if args.wandb_api_key:
+                    wandb.log(message)
+
+                # Log the message to the logger every n queries
+                if int(metrics.n_total_queries) % args.log_every_n_query == 0:
+                    msg = json.dumps(message, ensure_ascii=False, indent=2)
+                    logger.info(msg)
+
+                pbar.update(1)  # Update the progress bar
+
+    return metrics, result_db_path
+
+
+@exception_handler
+async def start_server(args: Arguments) -> bool:
+    if args.api.startswith('local'):
+        #  start local server
+        server = threading.Thread(target=start_app, args=(copy.deepcopy(args), ), daemon=True)
+        server.start()
+
+        if args.dataset.startswith('speed_benchmark'):
+            args.url = 'http://127.0.0.1:8877/v1/completions'
+        else:
+            args.url = 'http://127.0.0.1:8877/v1/chat/completions'
+        args.model = os.path.basename(args.model)
+
+    if not await test_connection(args):
+        raise TimeoutError('Test connection failed')
+
+
+@exception_handler
+async def benchmark(args: Arguments) -> None:
+    if platform.system() != 'Windows':
+        loop = asyncio.get_running_loop()
+        add_signal_handlers(loop)
+
+    request_queue = asyncio.Queue()
+    benchmark_data_queue = asyncio.Queue()
+
+    async def create_send_request_tasks():
+        tasks: List[asyncio.Task] = []
+        for idx in range(args.parallel):
+            task = asyncio.create_task(send_requests_worker(idx, request_queue, benchmark_data_queue, args))
+            tasks.append(task)
+        return tasks
+
+    async def run_tasks():
+        await start_server(args)
+
+        dispatch_task = asyncio.create_task(dispatch_requests_worker(request_queue, args))
+        statistic_benchmark_metric_task = asyncio.create_task(
+            statistic_benchmark_metric_worker(benchmark_data_queue, args))
+        send_request_tasks = await create_send_request_tasks()
+
+        expected_number_of_queries = await dispatch_task
+        await request_queue.join()
+        query_send_completed_event.set()
+
+        await asyncio.gather(*send_request_tasks, return_exceptions=True)
+        await benchmark_data_queue.join()
+        data_process_completed_event.set()
+
+        metrics, result_db_path = await statistic_benchmark_metric_task
+        summary_result(args, metrics, expected_number_of_queries, result_db_path)
+
+        await asyncio.sleep(0.250)
+
+    await run_tasks()