evalscope 0.6.1__py3-none-any.whl → 0.7.0__py3-none-any.whl

evalscope/perf/main.py

@@ -0,0 +1,35 @@
+import asyncio
+import platform
+from argparse import Namespace
+
+from evalscope.perf.arguments import Arguments, parse_args
+from evalscope.perf.benchmark import benchmark
+from evalscope.perf.utils.handler import add_signal_handlers
+from evalscope.utils.logger import get_logger
+from evalscope.utils.utils import seed_everything
+
+logger = get_logger()
+
+
+def run_perf_benchmark(args):
+    if isinstance(args, dict):
+        args = Arguments(**args)
+    elif isinstance(args, Namespace):
+        args = Arguments.from_args(args)
+    seed_everything(args.seed)
+
+    logger.info('Starting benchmark...')
+    logger.info(args)
+
+    if platform.system() == 'Windows':
+        asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
+
+    loop = asyncio.get_event_loop()
+    if platform.system() != 'Windows':
+        add_signal_handlers(loop)
+    loop.run_until_complete(benchmark(args))
+
+
+if __name__ == '__main__':
+    args = Arguments.from_args(parse_args())
+    run_perf_benchmark(args)

evalscope/perf/plugin/__init__.py

@@ -0,0 +1,2 @@
+from evalscope.perf.plugin.api import *
+from evalscope.perf.plugin.datasets import *

evalscope/perf/plugin/api/__init__.py

@@ -0,0 +1,3 @@
+from evalscope.perf.plugin.api.custom_api import CustomPlugin
+from evalscope.perf.plugin.api.dashscope_api import DashScopeApiPlugin
+from evalscope.perf.plugin.api.openai_api import OpenaiPlugin

evalscope/perf/{api_plugin_base.py → plugin/api/base.py}

@@ -1,14 +1,16 @@
 from abc import abstractmethod
 from typing import Any, Dict, List, Tuple
 
-from evalscope.perf.query_parameters import QueryParameters
+from evalscope.perf.arguments import Arguments
+
 
 class ApiPluginBase:
+
     def __init__(self, model_path: str) -> None:
         self.model_path = model_path
-        
+
     @abstractmethod
-    def build_request(self, messages: List[Dict], param: QueryParameters)->Dict:
+    def build_request(self, messages: List[Dict], param: Arguments) -> Dict:
         """Build a api request body.
 
         Args:
@@ -22,39 +24,36 @@ class ApiPluginBase:
             Dict: The api request body.
         """
         raise NotImplementedError
-    
+
     @abstractmethod
-    def parse_responses(self, 
-                        responses: List, 
-                        request: Any=None,
-                        **kwargs:Any) -> Tuple[int, int]:
+    def parse_responses(self, responses: List, request: Any = None, **kwargs: Any) -> Tuple[int, int]:
         """Parser responses and return number of request and response tokens.
 
         Args:
             responses (List[bytes]): List of http response body, for stream output,
-                there are multiple responses, each is bytes, for general only one. 
+                there are multiple responses, each is bytes, for general only one.
             request (Any): The request body.
 
         Returns:
             Tuple: (Number of prompt_tokens and number of completion_tokens).
         """
-        raise NotImplementedError  
+        raise NotImplementedError
 
     @staticmethod
     def replace_values(input_json: Any, model: str, prompt: str):
-        if isinstance(input_json, dict):  
+        if isinstance(input_json, dict):
             for key, value in input_json.items():
                 if isinstance(value, str):
-                    input_json[key] = value.replace("%m", model).replace("%p", prompt)
-                else:                    
-                    ApiPluginBase.replace_values(value, model, prompt)  
-        elif isinstance(input_json, list): 
+                    input_json[key] = value.replace('%m', model).replace('%p', prompt)
+                else:
+                    ApiPluginBase.replace_values(value, model, prompt)
+        elif isinstance(input_json, list):
             for idx, item in enumerate(input_json):
                 if isinstance(item, str):
-                    input_json[idx] = item.replace("%m", model).replace("%p", prompt)
+                    input_json[idx] = item.replace('%m', model).replace('%p', prompt)
                 else:
                     ApiPluginBase.replace_values(item, model, prompt)
         elif isinstance(input_json, str):
-            input_json = input_json.replace("%m", model).replace("%p", prompt)
+            input_json = input_json.replace('%m', model).replace('%p', prompt)
         else:
-            pass
+            pass

evalscope/perf/{custom_api.py → plugin/api/custom_api.py}

@@ -1,19 +1,26 @@
 from typing import Any, Dict, Iterator, List
+
 import json
-from evalscope.perf.api_plugin_base import ApiPluginBase
 from transformers import AutoTokenizer
-from evalscope.perf.plugin_registry import register_api
-from evalscope.perf.query_parameters import QueryParameters
 
-@register_api("custom")
+from evalscope.perf.arguments import Arguments
+from evalscope.perf.plugin.api.base import ApiPluginBase
+from evalscope.perf.plugin.registry import register_api
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+
+@register_api('custom')
 class CustomPlugin(ApiPluginBase):
     """Support tensorrt-llm triton server
     """
+
     def __init__(self, mode_path: str):
         """Init the plugin
 
         Args:
-            mode_path (str): The model path, we use the tokenizer 
+            mode_path (str): The model path, we use the tokenizer
                 weight in the model to calculate the number of the
                 input and output tokens.
         """
@@ -23,12 +30,12 @@ class CustomPlugin(ApiPluginBase):
         else:
             self.tokenizer = None
 
-    def build_request(self, messages: List[Dict], param: QueryParameters) -> Dict:
+    def build_request(self, messages: List[Dict], param: Arguments) -> Dict:
         """Build the openai format request based on prompt, dataset
 
         Args:
             message (Dict): The basic message to generator query.
-            param (QueryParameters): The query parameters.
+            param (Arguments): The query parameters.
 
         Raises:
             Exception: NotImplemented
@@ -41,8 +48,8 @@ class CustomPlugin(ApiPluginBase):
             ApiPluginBase.replace_values(query, param.model, messages[0]['content'])
             return query
         except Exception as e:
-            print(e)
-            print('Prompt: %s invalidate!'%messages)
+            logger.exception(e)
+            logger.error('Prompt: %s invalidate!' % messages)
             return None
 
     def parse_responses(self, responses, request: Any = None, **kwargs) -> Dict:
@@ -53,7 +60,7 @@ class CustomPlugin(ApiPluginBase):
 
         Args:
             responses (List[bytes]): List of http response body, for stream output,
-                there are multiple responses, for general only one. 
+                there are multiple responses, for general only one.
             kwargs: (Any): The command line --parameter content.
         Returns:
             Tuple: Return number of prompt token and number of completion tokens.
@@ -63,15 +70,15 @@ class CustomPlugin(ApiPluginBase):
         input_tokens = None
         output_tokens = None
         for response in responses:
-            js = json.loads(response)
+            data = json.loads(response)
             # {"context_logits":0.0,"cum_log_probs":0.0,"generation_logits":0.0,"model_name":"ensemble",
             # "model_version":"1","output_log_probs":[0.0,0.0,0.0,0.0,0.0],"sequence_end":false,"sequence_id":0,"sequence_start":false,"text_output":"性"}
-            if 'text_output' in js:
+            if 'text_output' in data:
                 if 0 in delta_contents:
-                    delta_contents[0].append(js['text_output'])
+                    delta_contents[0].append(data['text_output'])
                 else:
-                    delta_contents[0] = [js['text_output']]     
-        if input_tokens is None and output_tokens is None and self.tokenizer is not None:                
+                    delta_contents[0] = [data['text_output']]
+        if input_tokens is None and output_tokens is None and self.tokenizer is not None:
             input_tokens = 0
             output_tokens = 0
             for _, choice_contents in delta_contents.items():
@@ -80,8 +87,7 @@ class CustomPlugin(ApiPluginBase):
                 output_tokens += len(self.tokenizer.encode(full_response_content))
         elif input_tokens is None and output_tokens is None:  # no usage info get.
             input_tokens = 0
-            output_tokens = 0            
-        
+            output_tokens = 0
+            logger.warning('No usage info get.')
+
         return input_tokens, output_tokens
-        
-        

evalscope/perf/{dashscope_api.py → plugin/api/dashscope_api.py}

@@ -1,26 +1,30 @@
-
-from sys import maxsize
-import sys
+import os
 from typing import Any, Dict, Iterator, List
+
 import json
-from evalscope.perf.api_plugin_base import ApiPluginBase
 
-from evalscope.perf.plugin_registry import register_api
-from evalscope.perf.query_parameters import QueryParameters
+from evalscope.perf.arguments import Arguments
+from evalscope.perf.plugin.api.base import ApiPluginBase
+from evalscope.perf.plugin.registry import register_api
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
 
-@register_api("dashscope")
+@register_api('dashscope')
 class DashScopeApiPlugin(ApiPluginBase):
+
     def __init__(self, mode_path: str):
         """Init the plugin
 
         Args:
-            mode_path (str): The model path, we use the tokenizer 
+            mode_path (str): The model path, we use the tokenizer
                 weight in the model to calculate the number of the
                 input and output tokens.
         """
         super().__init__(model_path=mode_path)
-        
-    def build_request(self,messages: List[Dict], param: QueryParameters) -> Dict:
+
+    def build_request(self, messages: List[Dict], param: Arguments) -> Dict:
         """Build the openai format request based on prompt, dataset
 
         Args:
@@ -35,16 +39,26 @@ class DashScopeApiPlugin(ApiPluginBase):
         """
         try:
             if param.query_template is not None:
-                query = json.loads(param.query_template)
+                if param.query_template.startswith('@'):
+                    file_path = param.query_template[1:]
+                    if os.path.exists(file_path):
+                        with open(file_path, 'r') as file:
+                            query = json.load(file)
+                    else:
+                        raise FileNotFoundError(f'{file_path}')
+                else:
+                    query = json.loads(param.query_template)
+
                 query['input']['messages'] = messages  # replace template content with message.
                 return self.__compose_query_from_parameter(query, param)
             else:
                 query = {'messages': messages}
                 return self.__compose_query_from_parameter(query, param)
         except Exception as e:
-            print(e)
+            logger.exception(e)
             return None
-    def __compose_query_from_parameter(self, payload: Dict, param: QueryParameters):
+
+    def __compose_query_from_parameter(self, payload: Dict, param: Arguments):
         payload['model'] = param.model
         if 'parameters' not in payload:
             payload['parameters'] = {}
@@ -73,7 +87,7 @@ class DashScopeApiPlugin(ApiPluginBase):
 
         Args:
             responses (List[bytes]): List of http response body, for stream output,
-                there are multiple responses, for general only one. 
+                there are multiple responses, for general only one.
             kwargs: (Any): The command line --parameter content.
 
         Returns:

evalscope/perf/{openai_api.py → plugin/api/openai_api.py}

@@ -1,19 +1,26 @@
+import os
 from typing import Any, Dict, Iterator, List
+
 import json
-from evalscope.perf.api_plugin_base import ApiPluginBase
 from transformers import AutoTokenizer
-from evalscope.perf.plugin_registry import register_api
-from evalscope.perf.query_parameters import QueryParameters
 
-@register_api("openai")
+from evalscope.perf.arguments import Arguments
+from evalscope.perf.plugin.api.base import ApiPluginBase
+from evalscope.perf.plugin.registry import register_api
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+
+@register_api(['openai', 'local_vllm', 'local'])
 class OpenaiPlugin(ApiPluginBase):
-    """Base of openai interface.
-    """
+    """Base of openai interface."""
+
     def __init__(self, mode_path: str):
         """Init the plugin
 
         Args:
-            mode_path (str): The model path, we use the tokenizer 
+            mode_path (str): The model path, we use the tokenizer
                 weight in the model to calculate the number of the
                 input and output tokens.
         """
@@ -23,11 +30,11 @@ class OpenaiPlugin(ApiPluginBase):
         else:
             self.tokenizer = None
 
-    def build_request(self, messages: List[Dict], param: QueryParameters) -> Dict:
+    def build_request(self, messages: List[Dict] | str, param: Arguments) -> Dict:
         """Build the openai format request based on prompt, dataset
 
         Args:
-            message (Dict): The basic message to generator query.
+            message (List[Dict] | str): The basic message to generator query.
             param (QueryParameters): The query parameters.
 
         Raises:
@@ -38,22 +45,35 @@ class OpenaiPlugin(ApiPluginBase):
         """
         try:
             if param.query_template is not None:
-                query = json.loads(param.query_template)
+                if param.query_template.startswith('@'):
+                    file_path = param.query_template[1:]
+                    if os.path.exists(file_path):
+                        with open(file_path, 'r') as file:
+                            query = json.load(file)
+                    else:
+                        raise FileNotFoundError(f'{file_path}')
+                else:
+                    query = json.loads(param.query_template)
+
                 if 'stream' in query.keys():
                     param.stream = query['stream']
-                query['messages'] = messages   # replace template messages with input messages.
-                return self.__compose_query_from_parameter(query, param)
+                # replace template messages with input messages.
+                query['messages'] = messages
+            elif isinstance(messages, str):
+                query = {'prompt': messages}
             else:
                 query = {'messages': messages}
-                return self.__compose_query_from_parameter(query, param)
+            return self.__compose_query_from_parameter(query, param)
         except Exception as e:
-            print(e)
+            logger.exception(e)
             return None
-        
-    def __compose_query_from_parameter(self, payload: Dict, param: QueryParameters):
+
+    def __compose_query_from_parameter(self, payload: Dict, param: Arguments):
         payload['model'] = param.model
         if param.max_tokens is not None:
             payload['max_tokens'] = param.max_tokens
+        if param.min_tokens is not None:
+            payload['min_tokens'] = param.min_tokens
         if param.frequency_penalty is not None:
             payload['frequency_penalty'] = param.frequency_penalty
         if param.logprobs is not None:
@@ -66,7 +86,7 @@ class OpenaiPlugin(ApiPluginBase):
             payload['stop'] = param.stop
         if param.stream is not None and param.stream:
             payload['stream'] = param.stream
-            payload['stream_options'] = {"include_usage": True}
+            payload['stream_options'] = {'include_usage': True}
         if param.stop_token_ids is not None:
             payload['stop_token_ids'] = param.stop_token_ids
         if param.temperature is not None:
@@ -83,7 +103,7 @@ class OpenaiPlugin(ApiPluginBase):
 
         Args:
             responses (List[bytes]): List of http response body, for stream output,
-                there are multiple responses, for general only one. 
+                there are multiple responses, for general only one.
             kwargs: (Any): The command line --parameter content.
         Returns:
             Tuple: Return number of prompt token and number of completion tokens.
@@ -96,10 +116,15 @@ class OpenaiPlugin(ApiPluginBase):
             js = json.loads(response)
             if js['object'] == 'chat.completion':
                 for choice in js['choices']:
-                    delta_contents[choice['index']] = [choice['message']['content']]     
+                    delta_contents[choice['index']] = [choice['message']['content']]
+                input_tokens = js['usage']['prompt_tokens']
+                output_tokens = js['usage']['completion_tokens']
+            elif js['object'] == 'text_completion':
+                for choice in js['choices']:
+                    delta_contents[choice['index']] = [choice['text']]
                 input_tokens = js['usage']['prompt_tokens']
-                output_tokens = js['usage']['completion_tokens']                 
-            else:  # 'object' == "chat.completion.chunk":
+                output_tokens = js['usage']['completion_tokens']
+            elif js['object'] == 'chat.completion.chunk':
                 if 'choices' in js:
                     for choice in js['choices']:
                         if 'delta' in choice and 'index' in choice:
@@ -115,8 +140,8 @@ class OpenaiPlugin(ApiPluginBase):
                 # "choices":[],"usage":{"prompt_tokens":32,"total_tokens":384,"completion_tokens":352}}
                 if 'usage' in js and js['usage']:
                     input_tokens = js['usage']['prompt_tokens']
-                    output_tokens = js['usage']['completion_tokens']     
-        if input_tokens is None and output_tokens is None and self.tokenizer is not None:                
+                    output_tokens = js['usage']['completion_tokens']
+        if (input_tokens is None and output_tokens is None and self.tokenizer is not None):
             input_tokens = 0
             output_tokens = 0
             for idx, choice_contents in delta_contents.items():
@@ -125,8 +150,7 @@ class OpenaiPlugin(ApiPluginBase):
                 output_tokens += len(self.tokenizer.encode(full_response_content))
         elif input_tokens is None and output_tokens is None:  # no usage info get.
             input_tokens = 0
-            output_tokens = 0            
-        
+            output_tokens = 0
+            logger.warning('No usage info get.')
+
         return input_tokens, output_tokens
-        
-        

evalscope/perf/plugin/datasets/__init__.py

@@ -0,0 +1,6 @@
+from evalscope.perf.plugin.datasets.custom import CustomDatasetPlugin
+from evalscope.perf.plugin.datasets.flickr8k import FlickrDatasetPlugin
+from evalscope.perf.plugin.datasets.line_by_line import LineByLineDatasetPlugin
+from evalscope.perf.plugin.datasets.longalpaca import LongAlpacaDatasetPlugin
+from evalscope.perf.plugin.datasets.openqa import OpenqaDatasetPlugin
+from evalscope.perf.plugin.datasets.speed_benchmark import SpeedBenchmarkDatasetPlugin, SpeedBenchmarkLongDatasetPlugin

evalscope/perf/{dataset_plugin_base.py → plugin/datasets/base.py}

@@ -1,12 +1,15 @@
-from abc import abstractmethod
 import sys
+from abc import abstractmethod
 from typing import Any, Dict, Iterator, List, Tuple
+
 import json
 
-from evalscope.perf.query_parameters import QueryParameters
+from evalscope.perf.arguments import Arguments
+
 
 class DatasetPluginBase:
-    def __init__(self, query_parameters: QueryParameters):
+
+    def __init__(self, query_parameters: Arguments):
         """Build data set plugin
 
         Args:
@@ -21,9 +24,9 @@ class DatasetPluginBase:
 
     def __iter__(self):
         return self.build_messages()
-    
+
     @abstractmethod
-    def build_messages(self)->Iterator[List[Dict]]:
+    def build_messages(self) -> Iterator[List[Dict]]:
         """Build the request.
 
         Raises:
@@ -33,8 +36,8 @@ class DatasetPluginBase:
             Iterator[List[Dict]]: Yield request messages.
         """
         raise NotImplementedError
-    
-    def dataset_line_by_line(self, dataset: str)->Iterator[str]:
+
+    def dataset_line_by_line(self, dataset: str) -> Iterator[str]:
         """Get content line by line of dataset.
 
         Args:
@@ -46,8 +49,8 @@ class DatasetPluginBase:
         with open(dataset, 'r', encoding='utf-8') as f:
             for line in f:
                 yield line
-    
-    def dataset_json_list(self, dataset: str)->Iterator[Dict]:
+
+    def dataset_json_list(self, dataset: str) -> Iterator[Dict]:
         """Read data from file which is list of requests.
            Sample: https://huggingface.co/datasets/Yukang/LongAlpaca-12k
 
@@ -61,4 +64,4 @@ class DatasetPluginBase:
             content = f.read()
         data = json.loads(content)
         for item in data:
-            yield item      
+            yield item

evalscope/perf/plugin/datasets/custom.py

@@ -0,0 +1,21 @@
+from typing import Dict, Iterator, List
+
+from evalscope.perf.arguments import Arguments
+from evalscope.perf.plugin.datasets.base import DatasetPluginBase
+from evalscope.perf.plugin.registry import register_dataset
+
+
+@register_dataset('custom')
+class CustomDatasetPlugin(DatasetPluginBase):
+    """Read dataset and return prompt.
+    """
+
+    def __init__(self, query_parameters: Arguments):
+        super().__init__(query_parameters)
+
+    def build_messages(self) -> Iterator[List[Dict]]:
+        for item in self.dataset_line_by_line(self.query_parameters.dataset_path):
+            prompt = item.strip()
+            if len(prompt) > self.query_parameters.min_prompt_length and len(
+                    prompt) < self.query_parameters.max_prompt_length:
+                yield [{'role': 'user', 'content': prompt}]

evalscope/perf/plugin/datasets/flickr8k.py

@@ -0,0 +1,51 @@
+import base64
+from io import BytesIO
+from typing import Any, Dict, Iterator, List
+
+from modelscope.msdatasets import MsDataset
+from PIL import Image
+
+from evalscope.perf.arguments import Arguments
+from evalscope.perf.plugin.datasets.base import DatasetPluginBase
+from evalscope.perf.plugin.registry import register_dataset
+
+
+def PIL_to_base64(image: Image.Image) -> str:
+    buffered = BytesIO()
+    image.save(buffered, format='JPEG')
+    img_str = base64.b64encode(buffered.getvalue()).decode('utf-8')
+    return img_str
+
+
+@register_dataset('flickr8k')
+class FlickrDatasetPlugin(DatasetPluginBase):
+    """Read dataset and return prompt.
+    Datasets: https://www.modelscope.cn/datasets/clip-benchmark/wds_flickr8k/files
+    """
+
+    def __init__(self, query_parameters: Arguments):
+        super().__init__(query_parameters)
+
+    def build_messages(self) -> Iterator[List[Dict]]:
+        dataset = MsDataset.load('clip-benchmark/wds_flickr8k', split='test')
+
+        for item in dataset:
+            pil_image = item['jpg']
+            base64_iamge = PIL_to_base64(pil_image)
+
+            yield [{
+                'role':
+                'user',
+                'content': [
+                    {
+                        'type': 'text',
+                        'text': 'Describe the image'
+                    },
+                    {
+                        'type': 'image_url',
+                        'image_url': {
+                            'url': f'data:image/jpeg;base64,{base64_iamge}',
+                        }
+                    },
+                ],
+            }]

evalscope/perf/{datasets → plugin/datasets}/line_by_line.py

@@ -1,18 +1,22 @@
 import sys
 from typing import Dict, Iterator, List
-from evalscope.perf.dataset_plugin_base import DatasetPluginBase
-from evalscope.perf.plugin_registry import register_dataset
-from evalscope.perf.query_parameters import QueryParameters
+
+from evalscope.perf.arguments import Arguments
+from evalscope.perf.plugin.datasets.base import DatasetPluginBase
+from evalscope.perf.plugin.registry import register_dataset
+
 
 @register_dataset('line_by_line')
 class LineByLineDatasetPlugin(DatasetPluginBase):
     """Read dataset and return prompt.
     """
-    def __init__(self, query_parameters: QueryParameters):
+
+    def __init__(self, query_parameters: Arguments):
         super().__init__(query_parameters)
 
     def build_messages(self) -> Iterator[List[Dict]]:
         for item in self.dataset_line_by_line(self.query_parameters.dataset_path):
             prompt = item.strip()
-            if len(prompt) > self.query_parameters.min_prompt_length and len(prompt) < self.query_parameters.max_prompt_length:
+            if len(prompt) > self.query_parameters.min_prompt_length and len(
+                    prompt) < self.query_parameters.max_prompt_length:
                 yield [{'role': 'user', 'content': prompt}]

evalscope/perf/plugin/datasets/longalpaca.py

@@ -0,0 +1,28 @@
+from typing import Any, Dict, Iterator, List
+
+from modelscope import MsDataset
+
+from evalscope.perf.arguments import Arguments
+from evalscope.perf.plugin.datasets.base import DatasetPluginBase
+from evalscope.perf.plugin.registry import register_dataset
+
+
+@register_dataset('longalpaca')
+class LongAlpacaDatasetPlugin(DatasetPluginBase):
+    """Read data from file which is list of requests.
+           Sample: https://www.modelscope.cn/datasets/AI-ModelScope/LongAlpaca-12k/files
+    """
+
+    def __init__(self, query_parameters: Arguments):
+        super().__init__(query_parameters)
+
+    def build_messages(self) -> Iterator[List[Dict]]:
+        if not self.query_parameters.dataset_path:
+            ds = MsDataset.load('AI-ModelScope/LongAlpaca-12k', subset_name='default', split='train')
+        else:
+            ds = self.dataset_json_list(self.query_parameters.dataset_path)
+        for item in ds:
+            prompt = item['instruction'].strip()
+            if len(prompt) > self.query_parameters.min_prompt_length and len(
+                    prompt) < self.query_parameters.max_prompt_length:
+                yield [{'role': 'user', 'content': prompt}]