evalscope 0.10.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

evalscope/perf/plugin/api/custom_api.py

@@ -1,9 +1,11 @@
+import aiohttp
 import json
-from typing import Any, Dict, Iterator, List
+from typing import Any, AsyncGenerator, Dict, List, Tuple, Union
 
 from evalscope.perf.arguments import Arguments
 from evalscope.perf.plugin.api.base import ApiPluginBase
 from evalscope.perf.plugin.registry import register_api
+from evalscope.perf.utils.benchmark_util import BenchmarkData
 from evalscope.utils.logger import get_logger
 
 logger = get_logger()
@@ -11,82 +13,193 @@ logger = get_logger()
 
 @register_api('custom')
 class CustomPlugin(ApiPluginBase):
-    """Support tensorrt-llm triton server
+    """Support custom API implementations.
+
+    This class serves as a template for users to implement their own API plugins.
+    By extending this class, users can connect to any LLM API with custom request
+    and response formats.
     """
 
-    def __init__(self, mode_path: str):
-        """Init the plugin
+    def __init__(self, param: Arguments):
+        """Initialize the plugin with the provided parameters.
 
         Args:
-            mode_path (str): The model path, we use the tokenizer
-                weight in the model to calculate the number of the
-                input and output tokens.
+            param (Arguments): Configuration parameters for the plugin, including:
+                - tokenizer_path: Path to the tokenizer for token counting
+                - model: Name of the model to use
+                - Other request parameters like max_tokens, temperature, etc.
         """
-        super().__init__(model_path=mode_path)
-        if mode_path is not None:
-            from transformers import AutoTokenizer
-            self.tokenizer = AutoTokenizer.from_pretrained(mode_path)
+        super().__init__(param=param)
+        if param.tokenizer_path is not None:
+            from modelscope import AutoTokenizer
+            self.tokenizer = AutoTokenizer.from_pretrained(param.tokenizer_path)
         else:
             self.tokenizer = None
 
-    def build_request(self, messages: List[Dict], param: Arguments) -> Dict:
-        """Build the openai format request based on prompt, dataset
+    def build_request(self, messages: Union[List[Dict], str], param: Arguments = None) -> Dict:
+        """Build a custom API request body based on the input messages and parameters.
 
-        Args:
-            message (Dict): The basic message to generator query.
-            param (Arguments): The query parameters.
+        This method formats the input messages into the expected request format
+        for your custom API.
 
-        Raises:
-            Exception: NotImplemented
+        Args:
+            messages (Union[List[Dict], str]): The input messages to include in the request.
+                Could be a list of message dictionaries (for chat models) or a string (for completion models).
+            param (Arguments, optional): Request parameters. Defaults to self.param.
 
         Returns:
-            Dict: The request body. None if prompt format is error.
+            Dict: A properly formatted request body for your custom API.
         """
+        param = param or self.param
         try:
-            query = json.loads(param.query_template)
-            ApiPluginBase.replace_values(query, param.model, messages[0]['content'])
-            return query
+            # Create a default query format if no template is provided
+            if isinstance(messages, str):
+                query = {'input_text': messages}
+            else:
+                query = {'messages': messages}
+
+            # Add model parameters to the request
+            return self._add_parameters_to_request(query, param)
         except Exception as e:
             logger.exception(e)
-            logger.error('Prompt: %s invalidate!' % messages)
             return None
 
-    def parse_responses(self, responses, request: Any = None, **kwargs) -> Dict:
-        """Parser responses and return number of request and response tokens.
-           sample of the output delta:
-           {"id":"4","object":"chat.completion.chunk","created":1714030870,"model":"llama3","choices":[{"index":0,"delta":{"role":"assistant","content":""},"logprobs":null,"finish_reason":null}]}
+    def _add_parameters_to_request(self, payload: Dict, param: Arguments) -> Dict:
+        """Add model parameters to the request payload.
+
+        This helper method adds various parameters like temperature, max_tokens, etc.
+        to the request based on what your custom API supports.
+
+        Args:
+            payload (Dict): The base request payload.
+            param (Arguments): The parameters to add.
+
+        Returns:
+            Dict: The request payload with added parameters.
+        """
+        # Add the model name
+        payload['model'] = param.model
+
+        # Add various parameters if they are provided
+        if param.max_tokens is not None:
+            payload['max_tokens'] = param.max_tokens
+        if param.temperature is not None:
+            payload['temperature'] = param.temperature
+        if param.top_p is not None:
+            payload['top_p'] = param.top_p
+        if param.top_k is not None:
+            payload['top_k'] = param.top_k
+        if param.stream is not None:
+            payload['stream'] = param.stream
+            payload['stream_options'] = {'include_usage': True}
+
+        # Add any extra arguments passed via command line
+        if param.extra_args is not None:
+            payload.update(param.extra_args)
+
+        return payload
+
+    def parse_responses(self, responses: List[Dict], request: str = None, **kwargs) -> Tuple[int, int]:
+        """Parse API responses and return token counts.
+
+        This method extracts the number of input and output tokens from the API responses.
+        Different APIs may return this information in different formats, or you may need
+        to calculate it using a tokenizer.
+
+        Args:
+            responses (List[Dict]): List of API response strings.
+            request (str, optional): The original request, which might be needed for token calculation.
+            **kwargs: Additional arguments.
+
+        Returns:
+            Tuple[int, int]: (input_tokens, output_tokens) - The number of tokens in the prompt and completion.
+        """
+        try:
+            # Example 1: Try to get token counts from the API response
+            last_response = json.loads(responses[-1])
+
+            # If the API provides token usage information
+            if 'usage' in last_response and last_response['usage']:
+                input_tokens = last_response['usage'].get('prompt_tokens', 0)
+                output_tokens = last_response['usage'].get('completion_tokens', 0)
+                return input_tokens, output_tokens
+
+            # Example 2: Calculate tokens using the tokenizer if no usage info is available
+            if self.tokenizer is not None:
+                input_text = ''
+                output_text = ''
+
+                # Extract input text from the request
+                if request and 'messages' in request:
+                    # For chat API
+                    input_text = ' '.join([msg['content'] for msg in request['messages']])
+                elif request and 'input_text' in request:
+                    # For completion API
+                    input_text = request['input_text']
+
+                # Extract output text from the response
+                for response in responses:
+                    js = json.loads(response)
+                    if 'choices' in js:
+                        for choice in js['choices']:
+                            if 'message' in choice and 'content' in choice['message']:
+                                output_text += choice['message']['content']
+                            elif 'text' in choice:
+                                output_text += choice['text']
+
+                # Count tokens
+                input_tokens = len(self.tokenizer.encode(input_text))
+                output_tokens = len(self.tokenizer.encode(output_text))
+                return input_tokens, output_tokens
+
+            # If no usage information and no tokenizer, raise an error
+            raise ValueError(
+                'Cannot determine token counts: no usage information in response and no tokenizer provided.'
+            )
 
+        except Exception as e:
+            logger.error(f'Error parsing responses: {e}')
+            return 0, 0
+
+    async def process_request(
+        self, client_session: aiohttp.ClientSession, url: str, headers: Dict, body: Dict
+    ) -> BenchmarkData:
+        """Process the HTTP request and handle the response.
+
+        This method handles sending the request to your API and processing the response,
+        including handling streaming responses if supported.
 
         Args:
-            responses (List[bytes]): List of http response body, for stream output,
-                there are multiple responses, for general only one.
-            kwargs: (Any): The command line --parameter content.
+            client_session (aiohttp.ClientSession): The aiohttp client session.
+            url (str): The API endpoint URL.
+            headers (Dict): The request headers.
+            body (Dict): The request body.
+
         Returns:
-            Tuple: Return number of prompt token and number of completion tokens.
+            BenchmarkData: The benchmark data including response and timing info.
         """
-        full_response_content = ''
-        delta_contents = {}
-        input_tokens = None
-        output_tokens = None
-        for response in responses:
-            data = json.loads(response)
-            # {"context_logits":0.0,"cum_log_probs":0.0,"generation_logits":0.0,"model_name":"ensemble",
-            # "model_version":"1","output_log_probs":[0.0,0.0,0.0,0.0,0.0],"sequence_end":false,"sequence_id":0,"sequence_start":false,"text_output":"性"}
-            if 'text_output' in data:
-                if 0 in delta_contents:
-                    delta_contents[0].append(data['text_output'])
-                else:
-                    delta_contents[0] = [data['text_output']]
-        if input_tokens is None and output_tokens is None and self.tokenizer is not None:
-            input_tokens = 0
-            output_tokens = 0
-            for _, choice_contents in delta_contents.items():
-                full_response_content = ''.join([m for m in choice_contents])
-                input_tokens += len(self.tokenizer.encode(request['text_input']))
-                output_tokens += len(self.tokenizer.encode(full_response_content))
-        elif input_tokens is None and output_tokens is None:  # no usage info get.
-            input_tokens = 0
-            output_tokens = 0
-            logger.warning('No usage info get.')
-
-        return input_tokens, output_tokens
+        raise NotImplementedError(
+            'The `process_request` method must be implemented in a subclass. '
+            'For OpenAI-compatible APIs, consider inheriting from `DefaultApiPlugin` to reuse the default implementation.'  # noqa: E501
+        )
+
+
+if __name__ == '__main__':
+    # Example usage of the CustomPlugin
+    from dotenv import dotenv_values
+    env = dotenv_values('.env')
+
+    from evalscope.perf.arguments import Arguments
+    from evalscope.perf.main import run_perf_benchmark
+
+    args = Arguments(
+        model='qwen2.5-7b-instruct',
+        url='https://dashscope.aliyuncs.com/compatible-mode/v1/chat/completions',
+        api_key=env.get('DASHSCOPE_API_KEY'),
+        api='custom',  # Use the custom API plugin registered above
+        dataset='openqa',
+        number=1,
+        max_tokens=10
+    )
+
+    run_perf_benchmark(args)

evalscope/perf/plugin/api/dashscope_api.py

@@ -13,17 +13,10 @@ logger = get_logger()
 @register_api('dashscope')
 class DashScopeApiPlugin(ApiPluginBase):
 
-    def __init__(self, mode_path: str):
-        """Init the plugin
+    def __init__(self, param: Arguments):
+        super().__init__(param)
 
-        Args:
-            mode_path (str): The model path, we use the tokenizer
-                weight in the model to calculate the number of the
-                input and output tokens.
-        """
-        super().__init__(model_path=mode_path)
-
-    def build_request(self, messages: List[Dict], param: Arguments) -> Dict:
+    def build_request(self, messages: List[Dict], param: Arguments = None) -> Dict:
         """Build the openai format request based on prompt, dataset
 
         Args:
@@ -36,6 +29,7 @@ class DashScopeApiPlugin(ApiPluginBase):
         Returns:
             Dict: The request body. None if prompt format is error.
         """
+        param = param or self.param
         try:
             if param.query_template is not None:
                 if param.query_template.startswith('@'):

evalscope/perf/plugin/api/default_api.py

@@ -0,0 +1,214 @@
+import aiohttp
+import codecs
+import json
+import sys
+import time
+import traceback
+from typing import Any, Dict
+
+from evalscope.perf.arguments import Arguments
+from evalscope.perf.plugin.api.base import ApiPluginBase
+from evalscope.perf.utils.benchmark_util import BenchmarkData
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+
+class StreamedResponseHandler:
+    """Handles streaming HTTP responses by accumulating chunks until complete
+    messages are available."""
+
+    def __init__(self):
+        self.buffer = ''
+        # Keep decoder state across chunks to handle split multibyte sequences
+        self.decoder = codecs.getincrementaldecoder('utf-8')()
+
+    def add_chunk(self, chunk_bytes: bytes) -> list[str]:
+        """Add a chunk of bytes to the buffer and return any complete
+        messages."""
+        # Use incremental decoding so incomplete multibyte sequences don't error
+        try:
+            chunk_str = self.decoder.decode(chunk_bytes, final=False)
+        except UnicodeDecodeError:
+            # Bad bytes: drop them and reset decoder state to avoid corruption
+            self.decoder.reset()
+            chunk_str = chunk_bytes.decode('utf-8', errors='ignore')
+        self.buffer += chunk_str
+
+        messages = []
+
+        # Split by double newlines (SSE message separator)
+        while '\n\n' in self.buffer:
+            message, self.buffer = self.buffer.split('\n\n', 1)
+            message = message.strip()
+            if message:
+                messages.append(message)
+
+        # if self.buffer is not empty, check if it is a complete message
+        # by removing data: prefix and check if it is a valid JSON
+        if self.buffer.startswith('data: '):
+            message_content = self.buffer.removeprefix('data: ').strip()
+            if message_content == '[DONE]':
+                messages.append(self.buffer.strip())
+                self.buffer = ''
+            elif message_content:
+                try:
+                    json.loads(message_content)
+                    messages.append(self.buffer.strip())
+                    self.buffer = ''
+                except json.JSONDecodeError:
+                    # Incomplete JSON, wait for more chunks.
+                    pass
+
+        return messages
+
+
+class DefaultApiPlugin(ApiPluginBase):
+    """Default implementation of API plugin with common HTTP handling methods."""
+
+    def __init__(self, param: Arguments):
+        super().__init__(param)
+
+    async def process_request(
+        self, client_session: aiohttp.ClientSession, url: str, headers: Dict, body: Dict
+    ) -> BenchmarkData:
+        """Process the HTTP request and handle the response.
+
+        Args:
+            client_session: The aiohttp client session
+            url: The request URL
+            headers: The request headers
+            body: The request body
+
+        Returns:
+            BenchmarkData: Aggregated benchmarking data for the request/response.
+        """
+        headers = {'Content-Type': 'application/json', **headers}
+        data = json.dumps(body, ensure_ascii=False)  # serialize to JSON
+
+        output = BenchmarkData()
+        ttft = 0.0
+        generated_text = ''
+        st = time.perf_counter()
+        output.start_time = st
+        output.request = data
+        most_recent_timestamp = st
+        try:
+            async with client_session.post(url=url, data=data, headers=headers) as response:
+                content_type = response.headers.get('Content-Type', '')
+                if response.status == 200:
+                    # Handle streaming responses (SSE)
+                    if 'text/event-stream' in content_type:
+                        handler = StreamedResponseHandler()
+                        async for chunk_bytes in response.content.iter_any():
+
+                            if not chunk_bytes:
+                                continue
+
+                            messages = handler.add_chunk(chunk_bytes)
+                            for message in messages:
+                                # NOTE: SSE comments (often used as pings) start with
+                                # a colon. These are not JSON data payload and should
+                                # be skipped.
+                                if message.startswith(':'):
+                                    continue
+
+                                chunk = message.removeprefix('data: ')
+
+                                if chunk != '[DONE]':
+                                    timestamp = time.perf_counter()
+                                    data = json.loads(chunk)
+
+                                    if choices := data.get('choices'):
+                                        content = choices[0]['delta'].get('content')
+                                        # First token
+                                        if ttft == 0.0:
+                                            ttft = timestamp - st
+                                            output.first_chunk_latency = ttft
+
+                                        # Decoding phase
+                                        else:
+                                            output.inter_chunk_latency.append(timestamp - most_recent_timestamp)
+
+                                        generated_text += content or ''
+                                        output.response_messages.append(data)
+                                    elif usage := data.get('usage'):
+                                        output.prompt_tokens = usage.get('prompt_tokens')
+                                        output.completion_tokens = usage.get('completion_tokens')
+
+                                    most_recent_timestamp = timestamp
+
+                        output.generated_text = generated_text
+                        output.success = True
+                        output.completed_time = most_recent_timestamp
+                        output.query_latency = most_recent_timestamp - st
+
+                    # Handle non-stream JSON responses
+                    elif 'application/json' in content_type or 'application/' in content_type:
+                        payload: Any
+                        try:
+                            payload = await response.json()
+                        except Exception:
+                            # Fallback to text if JSON parsing fails
+                            payload = await response.text()
+
+                        timestamp = time.perf_counter()
+                        output.completed_time = timestamp
+                        output.query_latency = timestamp - st
+                        # For non-stream, first chunk equals full latency
+                        output.first_chunk_latency = output.query_latency
+
+                        if isinstance(payload, dict):
+                            # Extract generated text from choices
+                            text = ''
+                            if choices := payload.get('choices'):
+                                first = choices[0] if choices else {}
+                                # Chat Completions format
+                                msg = first.get('message') or {}
+                                if isinstance(msg, dict) and msg.get('content') is not None:
+                                    text = msg.get('content') or ''
+                                else:
+                                    # Legacy Completions format
+                                    text = first.get('text') or ''
+                            generated_text = text
+
+                            # Extract usage if provided
+                            if usage := payload.get('usage'):
+                                output.prompt_tokens = usage.get('prompt_tokens')
+                                output.completion_tokens = usage.get('completion_tokens')
+
+                            output.response_messages.append(payload)
+                        else:
+                            generated_text = str(payload)
+
+                        output.generated_text = generated_text
+                        output.success = True
+
+                    else:
+                        # Unknown successful content-type: read as text
+                        raw = await response.text()
+                        timestamp = time.perf_counter()
+                        output.completed_time = timestamp
+                        output.query_latency = timestamp - st
+                        output.first_chunk_latency = output.query_latency
+                        output.generated_text = raw
+                        output.response_messages.append(raw)
+                        output.success = True
+                else:
+                    # Try to parse structured error, fallback to reason/text
+                    try:
+                        err_payload = await response.json()
+                        output.error = json.dumps(err_payload, ensure_ascii=False)
+                    except Exception:
+                        try:
+                            output.error = await response.text()
+                        except Exception:
+                            output.error = response.reason or ''
+                    output.success = False
+        except Exception:
+            output.success = False
+            exc_info = sys.exc_info()
+            output.error = ''.join(traceback.format_exception(*exc_info))
+            logger.error(output.error)
+
+        return output