evalscope 0.5.0__py3-none-any.whl

evalscope/perf/_logging.py

@@ -0,0 +1,32 @@
+import logging
+import os
+
+
+logger = logging.getLogger('perf')
+
+
+def enable_logging():
+    level = os.environ.get('LOGGING_LEVEL', 'info')
+    if level is not None:  # set logging level.
+        if level not in ['info', 'debug']:
+            # set logging level env, but invalid value, use default.
+            level = 'info'
+        if level == 'info':
+            logger.setLevel(logging.INFO)
+        else:
+            logger.setLevel(logging.DEBUG)
+        # set default logging handler
+        console_handler = logging.StreamHandler()
+        formatter = logging.Formatter(
+            '%(asctime)s - %(name)s - %(filename)s - %(funcName)s - %(lineno)d - %(levelname)s - %(message)s'  # noqa E501
+        )
+        #formatter = logging.Formatter(
+        #    '%(asctime)s - %(name)s - %(levelname)s - %(message)s'  # noqa E501
+        #)
+        console_handler.setFormatter(formatter)
+        logger.addHandler(console_handler)
+
+
+# in release disable dashscope log
+# you can enable dashscope log for debugger.
+enable_logging()

evalscope/perf/api_plugin_base.py

@@ -0,0 +1,60 @@
+from abc import abstractmethod
+from typing import Any, Dict, List, Tuple
+
+from evalscope.perf.query_parameters import QueryParameters
+
+class ApiPluginBase:
+    def __init__(self, model_path: str) -> None:
+        self.model_path = model_path
+        
+    @abstractmethod
+    def build_request(self, messages: List[Dict], param: QueryParameters)->Dict:
+        """Build a api request body.
+
+        Args:
+            messages (List[Dict]): The messages generated by dataset.
+            param (QueryParameters): The query parameters.
+
+        Raises:
+            NotImplementedError: Not implemented.
+
+        Returns:
+            Dict: The api request body.
+        """
+        raise NotImplementedError
+    
+    @abstractmethod
+    def parse_responses(self, 
+                        responses: List, 
+                        request: Any=None,
+                        **kwargs:Any) -> Tuple[int, int]:
+        """Parser responses and return number of request and response tokens.
+
+        Args:
+            responses (List[bytes]): List of http response body, for stream output,
+                there are multiple responses, each is bytes, for general only one. 
+            request (Any): The request body.
+
+        Returns:
+            Tuple: (Number of prompt_tokens and number of completion_tokens).
+        """
+        raise NotImplementedError  
+
+    @staticmethod
+    def replace_values(input_json: Any, model: str, prompt: str):
+        if isinstance(input_json, dict):  
+            for key, value in input_json.items():
+                if isinstance(value, str):
+                    input_json[key] = value.replace("%m", model).replace("%p", prompt)
+                else:                    
+                    ApiPluginBase.replace_values(value, model, prompt)  
+        elif isinstance(input_json, list): 
+            for idx, item in enumerate(input_json):
+                if isinstance(item, str):
+                    input_json[idx] = item.replace("%m", model).replace("%p", prompt)
+                else:
+                    ApiPluginBase.replace_values(item, model, prompt)
+        elif isinstance(input_json, str):
+            input_json = input_json.replace("%m", model).replace("%p", prompt)
+        else:
+            pass

evalscope/perf/custom_api.py

@@ -0,0 +1,87 @@
+from typing import Any, Dict, Iterator, List
+import json
+from evalscope.perf.api_plugin_base import ApiPluginBase
+from transformers import AutoTokenizer
+from evalscope.perf.plugin_registry import register_api
+from evalscope.perf.query_parameters import QueryParameters
+
+@register_api("custom")
+class CustomPlugin(ApiPluginBase):
+    """Support tensorrt-llm triton server
+    """
+    def __init__(self, mode_path: str):
+        """Init the plugin
+
+        Args:
+            mode_path (str): The model path, we use the tokenizer 
+                weight in the model to calculate the number of the
+                input and output tokens.
+        """
+        super().__init__(model_path=mode_path)
+        if mode_path is not None:
+            self.tokenizer = AutoTokenizer.from_pretrained(mode_path)
+        else:
+            self.tokenizer = None
+
+    def build_request(self, messages: List[Dict], param: QueryParameters) -> Dict:
+        """Build the openai format request based on prompt, dataset
+
+        Args:
+            message (Dict): The basic message to generator query.
+            param (QueryParameters): The query parameters.
+
+        Raises:
+            Exception: NotImplemented
+
+        Returns:
+            Dict: The request body. None if prompt format is error.
+        """
+        try:
+            query = json.loads(param.query_template)
+            ApiPluginBase.replace_values(query, param.model, messages[0]['content'])
+            return query
+        except Exception as e:
+            print(e)
+            print('Prompt: %s invalidate!'%messages)
+            return None
+
+    def parse_responses(self, responses, request: Any = None, **kwargs) -> Dict:
+        """Parser responses and return number of request and response tokens.
+           sample of the output delta:
+           {"id":"4","object":"chat.completion.chunk","created":1714030870,"model":"llama3","choices":[{"index":0,"delta":{"role":"assistant","content":""},"logprobs":null,"finish_reason":null}]}
+
+
+        Args:
+            responses (List[bytes]): List of http response body, for stream output,
+                there are multiple responses, for general only one. 
+            kwargs: (Any): The command line --parameter content.
+        Returns:
+            Tuple: Return number of prompt token and number of completion tokens.
+        """
+        full_response_content = ''
+        delta_contents = {}
+        input_tokens = None
+        output_tokens = None
+        for response in responses:
+            js = json.loads(response)
+            # {"context_logits":0.0,"cum_log_probs":0.0,"generation_logits":0.0,"model_name":"ensemble",
+            # "model_version":"1","output_log_probs":[0.0,0.0,0.0,0.0,0.0],"sequence_end":false,"sequence_id":0,"sequence_start":false,"text_output":"性"}
+            if 'text_output' in js:
+                if 0 in delta_contents:
+                    delta_contents[0].append(js['text_output'])
+                else:
+                    delta_contents[0] = [js['text_output']]     
+        if input_tokens is None and output_tokens is None and self.tokenizer is not None:                
+            input_tokens = 0
+            output_tokens = 0
+            for _, choice_contents in delta_contents.items():
+                full_response_content = ''.join([m for m in choice_contents])
+                input_tokens += len(self.tokenizer.encode(request['text_input']))
+                output_tokens += len(self.tokenizer.encode(full_response_content))
+        elif input_tokens is None and output_tokens is None:  # no usage info get.
+            input_tokens = 0
+            output_tokens = 0            
+        
+        return input_tokens, output_tokens
+        
+        

evalscope/perf/dashscope_api.py

@@ -0,0 +1,84 @@
+
+from sys import maxsize
+import sys
+from typing import Any, Dict, Iterator, List
+import json
+from evalscope.perf.api_plugin_base import ApiPluginBase
+
+from evalscope.perf.plugin_registry import register_api
+from evalscope.perf.query_parameters import QueryParameters
+
+@register_api("dashscope")
+class DashScopeApiPlugin(ApiPluginBase):
+    def __init__(self, mode_path: str):
+        """Init the plugin
+
+        Args:
+            mode_path (str): The model path, we use the tokenizer 
+                weight in the model to calculate the number of the
+                input and output tokens.
+        """
+        super().__init__(model_path=mode_path)
+        
+    def build_request(self,messages: List[Dict], param: QueryParameters) -> Dict:
+        """Build the openai format request based on prompt, dataset
+
+        Args:
+            messages (List[Dict]): The basic message to generator query.
+            param (QueryParameters): The query parameters.
+
+        Raises:
+            Exception: NotImplemented
+
+        Returns:
+            Dict: The request body. None if prompt format is error.
+        """
+        try:
+            if param.query_template is not None:
+                query = json.loads(param.query_template)
+                query['input']['messages'] = messages  # replace template content with message.
+                return self.__compose_query_from_parameter(query, param)
+            else:
+                query = {'messages': messages}
+                return self.__compose_query_from_parameter(query, param)
+        except Exception as e:
+            print(e)
+            return None
+    def __compose_query_from_parameter(self, payload: Dict, param: QueryParameters):
+        payload['model'] = param.model
+        if 'parameters' not in payload:
+            payload['parameters'] = {}
+        if param.max_tokens is not None:
+            payload['parameters']['max_tokens'] = param.max_tokens
+        if param.frequency_penalty is not None:
+            payload['parameters']['frequency_penalty'] = param.frequency_penalty
+        if param.logprobs is not None:
+            payload['parameters']['logprobs'] = param.logprobs
+        if param.n_choices is not None:
+            payload['parameters']['n'] = param.n_choices
+        if param.seed is not None:
+            payload['parameters']['seed'] = param.seed
+        if param.stop is not None:
+            payload['parameters']['stop'] = param.stop
+        if param.stream is not None and not param.stream:
+            payload['parameters']['stream'] = param.stream
+        if param.temperature is not None:
+            payload['parameters']['temperature'] = param.temperature
+        if param.top_p is not None:
+            payload['parameters']['top_p'] = param.top_p
+        return payload
+
+    def parse_responses(self, responses, **kwargs) -> Dict:
+        """Parser responses and return number of request and response tokens.
+
+        Args:
+            responses (List[bytes]): List of http response body, for stream output,
+                there are multiple responses, for general only one. 
+            kwargs: (Any): The command line --parameter content.
+
+        Returns:
+            Tuple: Return number of prompt token and number of completion tokens.
+        """
+        last_response = responses[-1]
+        js = json.loads(last_response)
+        return js['usage']['input_tokens'], js['usage']['output_tokens']

evalscope/perf/dataset_plugin_base.py

@@ -0,0 +1,64 @@
+from abc import abstractmethod
+import sys
+from typing import Any, Dict, Iterator, List, Tuple
+import json
+
+from evalscope.perf.query_parameters import QueryParameters
+
+class DatasetPluginBase:
+    def __init__(self, query_parameters: QueryParameters):
+        """Build data set plugin
+
+        Args:
+            dataset_path (str, optional): The input dataset path. Defaults to None.
+        """
+        self.query_parameters = query_parameters
+
+    def __next__(self):
+        for item in self.build_messages():
+            yield item
+        raise StopIteration
+
+    def __iter__(self):
+        return self.build_messages()
+    
+    @abstractmethod
+    def build_messages(self)->Iterator[List[Dict]]:
+        """Build the request.
+
+        Raises:
+            NotImplementedError: The request is not impletion.
+
+        Yields:
+            Iterator[List[Dict]]: Yield request messages.
+        """
+        raise NotImplementedError
+    
+    def dataset_line_by_line(self, dataset: str)->Iterator[str]:
+        """Get content line by line of dataset.
+
+        Args:
+            dataset (str): The dataset path.
+
+        Yields:
+            Iterator[str]: Each line of file.
+        """
+        with open(dataset, 'r', encoding='utf-8') as f:
+            for line in f:
+                yield line
+    
+    def dataset_json_list(self, dataset: str)->Iterator[Dict]:
+        """Read data from file which is list of requests.
+           Sample: https://huggingface.co/datasets/Yukang/LongAlpaca-12k
+
+        Args:
+            dataset (str): The dataset path.
+
+        Yields:
+            Iterator[Dict]: The each request object.
+        """
+        with open(dataset, 'r', encoding='utf-8') as f:
+            content = f.read()
+        data = json.loads(content)
+        for item in data:
+            yield item      

evalscope/perf/datasets/line_by_line.py

@@ -0,0 +1,18 @@
+import sys
+from typing import Dict, Iterator, List
+from evalscope.perf.dataset_plugin_base import DatasetPluginBase
+from evalscope.perf.plugin_registry import register_dataset
+from evalscope.perf.query_parameters import QueryParameters
+
+@register_dataset('line_by_line')
+class LineByLineDatasetPlugin(DatasetPluginBase):
+    """Read dataset and return prompt.
+    """
+    def __init__(self, query_parameters: QueryParameters):
+        super().__init__(query_parameters)
+
+    def build_messages(self) -> Iterator[List[Dict]]:
+        for item in self.dataset_line_by_line(self.query_parameters.dataset_path):
+            prompt = item.strip()
+            if len(prompt) > self.query_parameters.min_prompt_length and len(prompt) < self.query_parameters.max_prompt_length:
+                yield [{'role': 'user', 'content': prompt}]

evalscope/perf/datasets/longalpaca_12k.py

@@ -0,0 +1,20 @@
+import sys
+from typing import Any, Dict, Iterator, List
+from evalscope.perf.dataset_plugin_base import DatasetPluginBase
+
+from evalscope.perf.plugin_registry import register_dataset
+from evalscope.perf.query_parameters import QueryParameters
+
+@register_dataset('longalpaca')
+class LongAlpacaDatasetPlugin(DatasetPluginBase):
+    """Read data from file which is list of requests.
+           Sample: https://huggingface.co/datasets/Yukang/LongAlpaca-12k
+    """
+    def __init__(self, query_parameters: QueryParameters):
+        super().__init__(query_parameters)
+
+    def build_messages(self) -> Iterator[List[Dict]]:
+        for item in self.dataset_json_list(self.query_parameters.dataset_path):
+            prompt = item['instruction'].strip()
+            if len(prompt) > self.query_parameters.min_prompt_length and len(prompt) < self.query_parameters.max_prompt_length:
+                yield [{'role': 'user', 'content': prompt}]

evalscope/perf/datasets/openqa.py

@@ -0,0 +1,22 @@
+from sys import maxsize
+import sys
+from typing import Any, Dict, Iterator, List
+import json
+from evalscope.perf.dataset_plugin_base import DatasetPluginBase
+from evalscope.perf.plugin_registry import register_dataset
+from evalscope.perf.query_parameters import QueryParameters
+
+@register_dataset('openqa')
+class OpenqaDatasetPlugin(DatasetPluginBase):
+    """Read dataset and return prompt.
+        Datasets: https://huggingface.co/datasets/Hello-SimpleAI/HC3-Chinese/blob/main/open_qa.jsonl
+    """
+    def __init__(self, query_parameters: QueryParameters):
+        super().__init__(query_parameters)
+        
+    def build_messages(self) -> Iterator[List[Dict]]:
+        for item in self.dataset_line_by_line(self.query_parameters.dataset_path):
+            item = json.loads(item)
+            prompt = item['question'].strip()
+            if len(prompt) > self.query_parameters.min_prompt_length and len(prompt) < self.query_parameters.max_prompt_length:
+                yield [{'role': 'user', 'content': prompt}]

evalscope/perf/how_to_analysis_result.py

@@ -0,0 +1,24 @@
+import sqlite3
+import base64
+import pickle
+import json
+result_db_path = 'db_name.db'
+con = sqlite3.connect(result_db_path)
+query_sql = "SELECT request, response_messages, prompt_tokens, completion_tokens \
+                FROM result WHERE success='True'"
+# how to save base64.b64encode(pickle.dumps(benchmark_data["request"])).decode("ascii"), 
+with con:
+    rows = con.execute(query_sql).fetchall()
+    if len(rows) > 0:
+        for row in rows:
+            request = row[0]
+            responses = row[1]
+            request = base64.b64decode(request)
+            request = pickle.loads(request)
+            responses = base64.b64decode(responses)
+            responses = pickle.loads(responses)
+            response_content = ''
+            for response in responses:
+                response = json.loads(response)
+                response_content += response['choices'][0]['delta']['content']
+            print('prompt: %s, tokens: %s, completion: %s, tokens: %s' % (request['messages'][0]['content'], row[2], response_content, row[3]))