evalscope 0.12.0__py3-none-any.whl → 0.12.1__py3-none-any.whl

evalscope/models/chat_adapter.py

@@ -3,8 +3,10 @@ import time
 import torch
 from typing import List, Union
 
+from evalscope.constants import OutputType
 from evalscope.models.base_adapter import BaseModelAdapter
 from evalscope.models.local_model import LocalModel
+from evalscope.models.register import register_model_adapter
 from evalscope.utils.chat_service import ChatCompletionResponse, ChatCompletionResponseChoice, ChatMessage
 from evalscope.utils.logger import get_logger
 from evalscope.utils.model_utils import fix_do_sample_warning
@@ -12,6 +14,7 @@ from evalscope.utils.model_utils import fix_do_sample_warning
 logger = get_logger()
 
 
+@register_model_adapter(OutputType.GENERATION)
 class ChatGenerationModelAdapter(BaseModelAdapter):
     """
     Chat generation model adapter.

evalscope/models/choice_adapter.py

@@ -3,11 +3,14 @@ import time
 import torch
 from typing import List
 
+from evalscope.constants import OutputType
 from evalscope.models.base_adapter import BaseModelAdapter
 from evalscope.models.local_model import LocalModel
+from evalscope.models.register import register_model_adapter
 from evalscope.utils.chat_service import ChatCompletionResponse, ChatCompletionResponseChoice, ChatMessage
 
 
+@register_model_adapter(OutputType.MULTIPLE_CHOICE)
 class MultiChoiceModelAdapter(BaseModelAdapter):
     """ The multi-choice model adapter. """
 
@@ -110,6 +113,7 @@ class MultiChoiceModelAdapter(BaseModelAdapter):
         return log_probs, {'tokens': tokens}
 
 
+@register_model_adapter(OutputType.CONTINUOUS)
 class ContinuationLogitsModelAdapter(MultiChoiceModelAdapter):
     """
     Continuation-logits model adapter.

evalscope/models/custom_adapter.py

@@ -2,8 +2,10 @@ from typing import Any, Dict, List, Union
 
 from evalscope.models.base_adapter import BaseModelAdapter
 from evalscope.models.custom import CustomModel
+from evalscope.models.register import register_model_adapter
 
 
+@register_model_adapter('custom')
 class CustomModelAdapter(BaseModelAdapter):
 
     def __init__(self, custom_model: CustomModel, **kwargs):

evalscope/models/register.py

@@ -0,0 +1,28 @@
+MODEL_ADAPTERS = {}
+
+
+def register_model_adapter(name):
+    """
+    Decorator to register a model adapter with a given name.
+    :param name: The name of the model adapter.
+    """
+
+    def decorator(adapter):
+        if name in MODEL_ADAPTERS:
+            raise ValueError(f"Model adapter '{name}' is already registered.")
+        MODEL_ADAPTERS[name] = adapter
+        return adapter
+
+    return decorator
+
+
+def get_model_adapter(name):
+    """
+    Retrieve a registered model adapter by name.
+    :param name: The name of the model adapter.
+    :return: The model adapter class or function.
+    """
+    if name not in MODEL_ADAPTERS:
+        raise ValueError(
+            f"Model adapter '{name}' is not registered. Available model adapters: {list(MODEL_ADAPTERS.keys())}")
+    return MODEL_ADAPTERS[name]

evalscope/models/server_adapter.py

@@ -1,15 +1,18 @@
 import openai
 from collections import defaultdict
+from inspect import signature
 from openai.types.chat import ChatCompletion, ChatCompletionChunk
 from openai.types.chat.chat_completion import ChatCompletionMessage, Choice
 from typing import List, Optional, Union
 
 from evalscope.models.base_adapter import BaseModelAdapter
+from evalscope.models.register import register_model_adapter
 from evalscope.utils.logger import get_logger
 
 logger = get_logger()
 
 
+@register_model_adapter('server')
 class ServerModelAdapter(BaseModelAdapter):
     """
     Server model adapter to request remote API model and generate results.
@@ -30,6 +33,7 @@ class ServerModelAdapter(BaseModelAdapter):
             api_key=api_key,
             base_url=self.api_url,
         )
+        self.supported_params = self._get_supported_params()
 
         self.seed = kwargs.get('seed', None)
         self.timeout = kwargs.get('timeout', 60)
@@ -37,12 +41,16 @@ class ServerModelAdapter(BaseModelAdapter):
         self.model_cfg = {'api_url': api_url, 'model_id': model_id, 'api_key': api_key}
         super().__init__(model=None, model_cfg=self.model_cfg, **kwargs)
 
-    def predict(self, inputs: List[Union[str, dict, list]], infer_cfg: dict = None) -> List[dict]:
+    def _get_supported_params(self):
+        sig = signature(self.client.chat.completions.create)
+        return list(sig.parameters.keys())
+
+    def predict(self, inputs: List[dict], infer_cfg: dict = None) -> List[dict]:
         """
         Model prediction func.
 
         Args:
-            inputs (List[Union[str, dict, list]]): The input data.
+            inputs (List[dict]): The input data.
             infer_cfg (dict): Inference configuration.
 
         Returns:
@@ -104,34 +112,52 @@ class ServerModelAdapter(BaseModelAdapter):
             request_json['stream_options'] = {'include_usage': True}
 
         logger.debug(f'Request to remote API: {request_json}')
+
         return request_json
 
+    def _parse_extra_params(self, request_json):
+        api_params = {}
+        extra_body = {}
+        for key, value in request_json.items():
+            if key in self.supported_params:
+                api_params[key] = value
+            else:
+                extra_body[key] = value
+
+        if extra_body:
+            api_params['extra_body'] = extra_body
+        return api_params
+
     def send_request(self, request_json: dict) -> dict:
         try:
-            response = self.client.chat.completions.create(**request_json)
+            parsed_request = self._parse_extra_params(request_json)
+            response = self.client.chat.completions.create(**parsed_request)
 
-            if self.stream:
+            if response and self.stream:
                 response = self._collect_stream_response(response)
 
             return response.model_dump(exclude_unset=True)
         except Exception as e:
-            logger.error(f'Error when calling OpenAI API: {str(e)}')
+            logger.error(f'Error when calling remote API: {str(e)}')
             raise
 
     def _collect_stream_response(self, response_stream: List[ChatCompletionChunk]) -> ChatCompletion:
         collected_chunks = []
         collected_messages = defaultdict(list)
+        collected_reasoning = defaultdict(list)
 
         for chunk in response_stream:
             collected_chunks.append(chunk)
             for choice in chunk.choices:
+                if hasattr(choice.delta, 'reasoning_content') and choice.delta.reasoning_content is not None:
+                    collected_reasoning[choice.index].append(choice.delta.reasoning_content)
                 if choice.delta.content is not None:
                     collected_messages[choice.index].append(choice.delta.content)
 
         choices = []
         for index, messages in collected_messages.items():
             full_reply_content = ''.join(messages)
-
+            reasoning = ''.join(collected_reasoning[index])
             # use the finish_reason from the last chunk that generated this choice
             finish_reason = None
             for chunk in reversed(collected_chunks):
@@ -140,9 +166,10 @@ class ServerModelAdapter(BaseModelAdapter):
                     break
 
             choice = Choice(
-                finish_reason=finish_reason,
+                finish_reason=finish_reason or 'stop',
                 index=index,
-                message=ChatCompletionMessage(role='assistant', content=full_reply_content))
+                message=ChatCompletionMessage(
+                    role='assistant', content=full_reply_content, reasoning_content=reasoning))
             choices.append(choice)
 
         # build the final completion object

evalscope/perf/arguments.py

@@ -21,9 +21,9 @@ class Arguments:
     # Connection settings
     url: str = 'http://127.0.0.1:8877/v1/chat/completions'  # URL for the API connection
     headers: Dict[str, Any] = field(default_factory=dict)  # Custom headers
-    connect_timeout: int = 120  # Connection timeout in seconds
-    read_timeout: int = 120  # Read timeout in seconds
-    api_key: str = 'EMPTY'
+    connect_timeout: int = 600  # Connection timeout in seconds
+    read_timeout: int = 600  # Read timeout in seconds
+    api_key: Optional[str] = None
 
     # Performance and parallelism
     number: Optional[int] = None  # Number of requests to be made
@@ -125,7 +125,13 @@ class ParseKVAction(argparse.Action):
             setattr(namespace, self.dest, {})
         else:
             try:
-                kv_dict = dict(kv.split('=') for kv in values)
+                kv_dict = {}
+                for kv in values:
+                    parts = kv.split('=', 1)  # only split the first '='
+                    if len(parts) != 2:
+                        raise ValueError(f'Invalid key-value pair: {kv}')
+                    key, value = parts
+                    kv_dict[key.strip()] = value.strip()
                 setattr(namespace, self.dest, kv_dict)
             except ValueError as e:
                 parser.error(f'Error parsing key-value pairs: {e}')
@@ -144,9 +150,9 @@ def add_argument(parser: argparse.ArgumentParser):
     parser.add_argument('--url', type=str, default='http://127.0.0.1:8877/v1/chat/completions')
     parser.add_argument('--port', type=int, default=8877, help='The port for local inference')
     parser.add_argument('--headers', nargs='+', dest='headers', action=ParseKVAction, help='Extra HTTP headers')
-    parser.add_argument('--api-key', type=str, required=False, default='EMPTY', help='The API key for authentication')
-    parser.add_argument('--connect-timeout', type=int, default=120, help='The network connection timeout')
-    parser.add_argument('--read-timeout', type=int, default=120, help='The network read timeout')
+    parser.add_argument('--api-key', type=str, required=False, default=None, help='The API key for authentication')
+    parser.add_argument('--connect-timeout', type=int, default=600, help='The network connection timeout')
+    parser.add_argument('--read-timeout', type=int, default=600, help='The network read timeout')
 
     # Performance and parallelism
     parser.add_argument('-n', '--number', type=int, default=None, help='How many requests to be made')

evalscope/perf/http_client.py

@@ -23,10 +23,7 @@ class AioHttpClient:
         self.read_timeout = args.read_timeout
         self.connect_timeout = args.connect_timeout
         self.client = aiohttp.ClientSession(
-            timeout=aiohttp.ClientTimeout(
-                total=self.read_timeout + self.connect_timeout,
-                connect=self.connect_timeout,
-                sock_read=self.read_timeout),
+            timeout=aiohttp.ClientTimeout(connect=self.connect_timeout, sock_read=self.read_timeout),
             connector=aiohttp.TCPConnector(limit=1),
             trace_configs=[self._create_trace_config()] if args.debug else [])
 
@@ -102,6 +99,11 @@ class AioHttpClient:
             async with self.client.request('POST', url=self.url, data=data, headers=headers) as response:
                 async for rsp in self._handle_response(response):
                     yield rsp
+        except asyncio.TimeoutError:
+            logger.error(
+                f'TimeoutError: connect_timeout: {self.connect_timeout}, read_timeout: {self.read_timeout}. Please set longger timeout.'  # noqa: E501
+            )
+            yield (True, None, 'Timeout')
         except (aiohttp.ClientConnectorError, Exception) as e:
             logger.error(e)
             yield (True, None, e)

evalscope/perf/utils/analysis_result.py

@@ -3,7 +3,7 @@ import json
 import pickle
 import sqlite3
 
-result_db_path = '/mnt/data/data/user/maoyunlin.myl/eval-scope/outputs/qwen2.5_benchmark_20241111_160543.db'
+result_db_path = './outputs/qwen2.5_benchmark_20241111_160543.db'
 con = sqlite3.connect(result_db_path)
 query_sql = "SELECT request, response_messages, prompt_tokens, completion_tokens \
                 FROM result WHERE success='1'"

evalscope/report/app.py

@@ -125,6 +125,9 @@ def get_compare_report_df(acc_df: pd.DataFrame):
 
 
 def plot_single_report_scores(df: pd.DataFrame):
+    if df is None:
+        return None
+    logger.debug(f'df: {df}')
     plot = px.bar(df, x=df[ReportKey.dataset_name], y=df[ReportKey.score], text=df[ReportKey.score])
 
     width = 0.2 if len(df[ReportKey.dataset_name]) <= 5 else None

evalscope/report/combinator.py

@@ -57,8 +57,8 @@ class ReportsRecorder:
 
 
 if __name__ == '__main__':
-    report_dir_1 = '/mnt/data/data/user/maoyunlin.myl/eval-scope/outputs/20250117_151926'
-    # report_dir_2 = '/mnt/data/data/user/maoyunlin.myl/eval-scope/outputs/20250107_204445/reports'
+    report_dir_1 = './outputs/20250117_151926'
+    # report_dir_2 = './outputs/20250107_204445/reports'
 
     report_table = gen_table([report_dir_1])
     print(report_table)

evalscope/run.py

@@ -2,7 +2,7 @@
 """
 Run evaluation for LLMs.
 """
-import os.path
+import os
 from argparse import Namespace
 from datetime import datetime
 from typing import TYPE_CHECKING, List, Optional, Union
@@ -127,16 +127,17 @@ def create_evaluator(task_cfg: TaskConfig, dataset_name: str, outputs: OutputsSt
     from evalscope.models import initialize_model_adapter
 
     benchmark: BenchmarkMeta = Benchmark.get(dataset_name)
-    # Initialize data adapter
-    data_adapter = benchmark.get_data_adapter(config=task_cfg.dataset_args.get(dataset_name, {}))
 
     if dataset_name == DataCollection.NAME:
         # EvaluatorCollection is a collection of evaluators
         from evalscope.collections import EvaluatorCollection
+        data_adapter = benchmark.get_data_adapter(config=task_cfg.dataset_args.get(dataset_name, {}))
         return EvaluatorCollection(task_cfg, data_adapter, outputs)
 
     # Initialize model adapter
-    model_adapter = initialize_model_adapter(task_cfg, benchmark.model_adapter, base_model)
+    model_adapter = initialize_model_adapter(task_cfg, benchmark, base_model)
+    # Initialize data adapter
+    data_adapter = benchmark.get_data_adapter(config=task_cfg.dataset_args.get(dataset_name, {}))
 
     # update task_cfg.dataset_args
     task_cfg.dataset_args[dataset_name] = benchmark.to_string_dict()