evalscope 0.12.0__py3-none-any.whl → 0.13.0__py3-none-any.whl

evalscope/metrics/llm_judge.py

@@ -0,0 +1,104 @@
+import os
+import re
+from typing import Any, Dict, List, Optional
+
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+DEFAULT_PROMPT_TEMPLATE = """Your job is to look at a question, a gold target, and a predicted answer, and return a letter "A" or "B" to indicate whether the predicted answer is correct or incorrect.
+
+Question: {question}
+
+Reference Answer: {gold}
+
+Model Answer: {pred}
+
+Evaluate the model's answer based on correctness compared to the reference answer.
+Grade the predicted answer of this new question as one of:
+A: CORRECT
+B: INCORRECT
+
+Just return the letters "A" or "B", with no text around it.
+"""  # noqa: E501
+
+
+class LLMJudge:
+    """
+    A metric that uses LLM to judge the quality of model predictions by comparing them with reference answers.
+    """
+
+    def __init__(self,
+                 api_key: Optional[str] = None,
+                 api_url: Optional[str] = None,
+                 model_id: Optional[str] = None,
+                 system_prompt: Optional[str] = None,
+                 prompt_template: Optional[str] = None,
+                 generation_config: Optional[Dict[str, Any]] = None,
+                 **kwargs):
+        """
+        Initialize LLMJudge metric.
+
+        Args:
+            api_key (str, optional): API key for OpenAI or compatible service
+            api_base (str, optional): API base URL
+            model_id (str, optional): Model ID for LLM
+            system_prompt (str, optional): System prompt for the judge
+            prompt_template (str, optional): Prompt template for the judge
+            generation_config (dict, optional): Generation configuration for the judge
+        """
+        self.api_key = api_key or os.environ.get('OPENAI_API_KEY', 'EMPTY')
+        self.api_url = api_url or os.environ.get('OPENAI_API_BASE', 'https://api.openai.com/v1')
+        self.model_id = model_id or os.environ.get('LOCAL_LLM', 'gpt-3.5-turbo')
+        self.system_prompt = system_prompt or os.environ.get('JUDGE_SYSTEM_PROMPT', None)
+        self.prompt_template = prompt_template or os.environ.get('JUDGE_PROMPT_TEMPLATE', DEFAULT_PROMPT_TEMPLATE)
+        self.generation_config = generation_config
+
+        from evalscope.models.server_adapter import ServerModelAdapter
+
+        # Initialize ServerModelAdapter
+        self.server_adapter = ServerModelAdapter(api_url=self.api_url, model_id=self.model_id, api_key=self.api_key)
+
+    def __call__(self, prompt: str, system_prompt: Optional[str] = None) -> float:
+        """
+        Args:
+            prompt (str): The prompt to evaluate
+            system_prompt (str, optional): The system prompt to use for the evaluation
+        Returns:
+            float: The score of the evaluation
+        """
+        input_data = {'data': [prompt], 'system_prompt': system_prompt or self.system_prompt}
+
+        # Inference configuration
+        infer_cfg = {'temperature': 0.0, 'max_tokens': 1024}
+        if self.generation_config:
+            infer_cfg.update(self.generation_config)
+
+        try:
+            # Send request using ServerModelAdapter
+            response = self.server_adapter.process_single_input(input_data, infer_cfg)
+
+            # Extract content from response
+            llm_response = response.get('choices', [{}])[0].get('message', {}).get('content', '')
+            return llm_response
+        except Exception as e:
+            logger.error(f'Error during LLM evaluation: {e}')
+            return None
+
+    def build_prompt(self, pred: str, gold: str, question: Optional[str] = None):
+        if question is None:
+            question = 'Not provided'
+        return self.prompt_template.format(question=question, pred=pred, gold=gold)
+
+    def get_score(self, response: str) -> float:
+        if response is None:
+            return 0
+        match = re.search(r'(A|B)', response)
+        if match:
+            answer = match.group(0)
+            if answer == 'A':
+                return 1
+            elif answer == 'B':
+                return 0
+        else:
+            return 0

evalscope/metrics/named_metrics.py

@@ -35,6 +35,7 @@ metric_registry = MetricRegistry()
 metric_registry.register(Metric(name='AverageAccuracy', object=mean))
 metric_registry.register(Metric(name='WeightedAverageAccuracy', object=weighted_mean))
 metric_registry.register(Metric(name='AverageBLEU', object=mean))
+metric_registry.register(Metric(name='AverageRouge', object=mean))
 metric_registry.register(Metric(name='WeightedAverageBLEU', object=weighted_mean))
 metric_registry.register(Metric(name='AveragePass@1', object=mean))
 for k in range(1, 17):

evalscope/models/__init__.py

@@ -7,10 +7,11 @@ from evalscope.models.custom import CustomModel
 from evalscope.models.custom_adapter import CustomModelAdapter
 from evalscope.models.local_model import LocalModel, get_local_model
 from evalscope.models.model import BaseModel, ChatBaseModel, OpenAIModel
+from evalscope.models.register import get_model_adapter
 from evalscope.models.server_adapter import ServerModelAdapter
 
 __all__ = [
     'CustomModel', 'BaseModel', 'ChatBaseModel', 'OpenAIModel', 'BaseModelAdapter', 'ChatGenerationModelAdapter',
     'MultiChoiceModelAdapter', 'ContinuationLogitsModelAdapter', 'CustomModelAdapter', 'ServerModelAdapter',
-    'LocalModel', 'get_local_model', 'initialize_model_adapter'
+    'LocalModel', 'get_local_model', 'initialize_model_adapter', 'get_model_adapter'
 ]

evalscope/models/base_adapter.py

@@ -1,15 +1,21 @@
 import torch
 from abc import ABC, abstractmethod
-from typing import TYPE_CHECKING, Any, Optional, Union
+from typing import TYPE_CHECKING, Any, List, Optional, Union
 
-from evalscope.constants import EvalType
+from evalscope.constants import EvalType, OutputType
 from evalscope.models.custom import CustomModel
 from evalscope.models.local_model import LocalModel
+from evalscope.models.register import get_model_adapter, register_model_adapter
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
 
 if TYPE_CHECKING:
+    from evalscope.benchmarks import BenchmarkMeta
     from evalscope.config import TaskConfig
 
 
+@register_model_adapter('base')
 class BaseModelAdapter(ABC):
 
     def __init__(self, model: Optional[Union[LocalModel, CustomModel]], **kwargs):
@@ -33,7 +39,7 @@ class BaseModelAdapter(ABC):
         raise NotImplementedError
 
 
-def initialize_model_adapter(task_cfg: 'TaskConfig', model_adapter_cls: 'BaseModelAdapter', base_model: 'LocalModel'):
+def initialize_model_adapter(task_cfg: 'TaskConfig', benchmark: 'BenchmarkMeta', base_model: 'LocalModel'):
     """Initialize the model adapter based on the task configuration."""
     if task_cfg.dry_run:
         from evalscope.models.model import DummyChatModel
@@ -43,8 +49,14 @@ def initialize_model_adapter(task_cfg: 'TaskConfig', model_adapter_cls: 'BaseMod
             raise ValueError(f'Expected evalscope.models.custom.CustomModel, but got {type(task_cfg.model)}.')
         from evalscope.models import CustomModelAdapter
         return CustomModelAdapter(custom_model=task_cfg.model)
-    elif task_cfg.eval_type == EvalType.SERVICE:
+    elif task_cfg.eval_type == EvalType.SERVICE or task_cfg.api_url is not None:
         from evalscope.models import ServerModelAdapter
+
+        if benchmark.model_adapter in [OutputType.CONTINUOUS, OutputType.MULTIPLE_CHOICE]:
+            logger.warning('Output type is set to logits. This is not supported for service evaluation. '
+                           'Setting output type to generation by default.')
+            benchmark.model_adapter = OutputType.GENERATION
+
         return ServerModelAdapter(
             api_url=task_cfg.api_url,
             model_id=task_cfg.model,
@@ -54,5 +66,13 @@ def initialize_model_adapter(task_cfg: 'TaskConfig', model_adapter_cls: 'BaseMod
             stream=task_cfg.stream,
         )
     else:
-        return model_adapter_cls(
+        # for local model, we need to determine the model adapter class based on the output type
+        model_adapter_cls = benchmark.model_adapter
+        if model_adapter_cls not in benchmark.output_types:
+            logger.warning(f'Output type {model_adapter_cls} is not supported for benchmark {benchmark.name}. '
+                           f'Using {benchmark.output_types[0]} instead.')
+            model_adapter_cls = benchmark.output_types[0]
+
+        model_adapter = get_model_adapter(model_adapter_cls)
+        return model_adapter(
             model=base_model, generation_config=task_cfg.generation_config, chat_template=task_cfg.chat_template)

evalscope/models/chat_adapter.py

@@ -3,8 +3,10 @@ import time
 import torch
 from typing import List, Union
 
+from evalscope.constants import OutputType
 from evalscope.models.base_adapter import BaseModelAdapter
 from evalscope.models.local_model import LocalModel
+from evalscope.models.register import register_model_adapter
 from evalscope.utils.chat_service import ChatCompletionResponse, ChatCompletionResponseChoice, ChatMessage
 from evalscope.utils.logger import get_logger
 from evalscope.utils.model_utils import fix_do_sample_warning
@@ -12,6 +14,7 @@ from evalscope.utils.model_utils import fix_do_sample_warning
 logger = get_logger()
 
 
+@register_model_adapter(OutputType.GENERATION)
 class ChatGenerationModelAdapter(BaseModelAdapter):
     """
     Chat generation model adapter.

evalscope/models/choice_adapter.py

@@ -3,11 +3,14 @@ import time
 import torch
 from typing import List
 
+from evalscope.constants import OutputType
 from evalscope.models.base_adapter import BaseModelAdapter
 from evalscope.models.local_model import LocalModel
+from evalscope.models.register import register_model_adapter
 from evalscope.utils.chat_service import ChatCompletionResponse, ChatCompletionResponseChoice, ChatMessage
 
 
+@register_model_adapter(OutputType.MULTIPLE_CHOICE)
 class MultiChoiceModelAdapter(BaseModelAdapter):
     """ The multi-choice model adapter. """
 
@@ -110,6 +113,7 @@ class MultiChoiceModelAdapter(BaseModelAdapter):
         return log_probs, {'tokens': tokens}
 
 
+@register_model_adapter(OutputType.CONTINUOUS)
 class ContinuationLogitsModelAdapter(MultiChoiceModelAdapter):
     """
     Continuation-logits model adapter.

evalscope/models/custom_adapter.py

@@ -2,8 +2,10 @@ from typing import Any, Dict, List, Union
 
 from evalscope.models.base_adapter import BaseModelAdapter
 from evalscope.models.custom import CustomModel
+from evalscope.models.register import register_model_adapter
 
 
+@register_model_adapter('custom')
 class CustomModelAdapter(BaseModelAdapter):
 
     def __init__(self, custom_model: CustomModel, **kwargs):

evalscope/models/register.py

@@ -0,0 +1,28 @@
+MODEL_ADAPTERS = {}
+
+
+def register_model_adapter(name):
+    """
+    Decorator to register a model adapter with a given name.
+    :param name: The name of the model adapter.
+    """
+
+    def decorator(adapter):
+        if name in MODEL_ADAPTERS:
+            raise ValueError(f"Model adapter '{name}' is already registered.")
+        MODEL_ADAPTERS[name] = adapter
+        return adapter
+
+    return decorator
+
+
+def get_model_adapter(name):
+    """
+    Retrieve a registered model adapter by name.
+    :param name: The name of the model adapter.
+    :return: The model adapter class or function.
+    """
+    if name not in MODEL_ADAPTERS:
+        raise ValueError(
+            f"Model adapter '{name}' is not registered. Available model adapters: {list(MODEL_ADAPTERS.keys())}")
+    return MODEL_ADAPTERS[name]

evalscope/models/server_adapter.py

@@ -1,15 +1,18 @@
 import openai
 from collections import defaultdict
+from inspect import signature
 from openai.types.chat import ChatCompletion, ChatCompletionChunk
 from openai.types.chat.chat_completion import ChatCompletionMessage, Choice
 from typing import List, Optional, Union
 
 from evalscope.models.base_adapter import BaseModelAdapter
+from evalscope.models.register import register_model_adapter
 from evalscope.utils.logger import get_logger
 
 logger = get_logger()
 
 
+@register_model_adapter('server')
 class ServerModelAdapter(BaseModelAdapter):
     """
     Server model adapter to request remote API model and generate results.
@@ -30,6 +33,7 @@ class ServerModelAdapter(BaseModelAdapter):
             api_key=api_key,
             base_url=self.api_url,
         )
+        self.supported_params = self._get_supported_params()
 
         self.seed = kwargs.get('seed', None)
         self.timeout = kwargs.get('timeout', 60)
@@ -37,12 +41,16 @@ class ServerModelAdapter(BaseModelAdapter):
         self.model_cfg = {'api_url': api_url, 'model_id': model_id, 'api_key': api_key}
         super().__init__(model=None, model_cfg=self.model_cfg, **kwargs)
 
-    def predict(self, inputs: List[Union[str, dict, list]], infer_cfg: dict = None) -> List[dict]:
+    def _get_supported_params(self):
+        sig = signature(self.client.chat.completions.create)
+        return list(sig.parameters.keys())
+
+    def predict(self, inputs: List[dict], infer_cfg: dict = None) -> List[dict]:
         """
         Model prediction func.
 
         Args:
-            inputs (List[Union[str, dict, list]]): The input data.
+            inputs (List[dict]): The input data.
             infer_cfg (dict): Inference configuration.
 
         Returns:
@@ -104,34 +112,52 @@ class ServerModelAdapter(BaseModelAdapter):
             request_json['stream_options'] = {'include_usage': True}
 
         logger.debug(f'Request to remote API: {request_json}')
+
         return request_json
 
+    def _parse_extra_params(self, request_json):
+        api_params = {}
+        extra_body = {}
+        for key, value in request_json.items():
+            if key in self.supported_params:
+                api_params[key] = value
+            else:
+                extra_body[key] = value
+
+        if extra_body:
+            api_params['extra_body'] = extra_body
+        return api_params
+
     def send_request(self, request_json: dict) -> dict:
         try:
-            response = self.client.chat.completions.create(**request_json)
+            parsed_request = self._parse_extra_params(request_json)
+            response = self.client.chat.completions.create(**parsed_request)
 
-            if self.stream:
+            if response and self.stream:
                 response = self._collect_stream_response(response)
 
             return response.model_dump(exclude_unset=True)
         except Exception as e:
-            logger.error(f'Error when calling OpenAI API: {str(e)}')
+            logger.error(f'Error when calling remote API: {str(e)}')
             raise
 
     def _collect_stream_response(self, response_stream: List[ChatCompletionChunk]) -> ChatCompletion:
         collected_chunks = []
         collected_messages = defaultdict(list)
+        collected_reasoning = defaultdict(list)
 
         for chunk in response_stream:
             collected_chunks.append(chunk)
             for choice in chunk.choices:
+                if hasattr(choice.delta, 'reasoning_content') and choice.delta.reasoning_content is not None:
+                    collected_reasoning[choice.index].append(choice.delta.reasoning_content)
                 if choice.delta.content is not None:
                     collected_messages[choice.index].append(choice.delta.content)
 
         choices = []
         for index, messages in collected_messages.items():
             full_reply_content = ''.join(messages)
-
+            reasoning = ''.join(collected_reasoning[index])
             # use the finish_reason from the last chunk that generated this choice
             finish_reason = None
             for chunk in reversed(collected_chunks):
@@ -140,9 +166,10 @@ class ServerModelAdapter(BaseModelAdapter):
                     break
 
             choice = Choice(
-                finish_reason=finish_reason,
+                finish_reason=finish_reason or 'stop',
                 index=index,
-                message=ChatCompletionMessage(role='assistant', content=full_reply_content))
+                message=ChatCompletionMessage(
+                    role='assistant', content=full_reply_content, reasoning_content=reasoning))
             choices.append(choice)
 
         # build the final completion object

evalscope/perf/arguments.py

@@ -21,9 +21,9 @@ class Arguments:
     # Connection settings
     url: str = 'http://127.0.0.1:8877/v1/chat/completions'  # URL for the API connection
     headers: Dict[str, Any] = field(default_factory=dict)  # Custom headers
-    connect_timeout: int = 120  # Connection timeout in seconds
-    read_timeout: int = 120  # Read timeout in seconds
-    api_key: str = 'EMPTY'
+    connect_timeout: int = 600  # Connection timeout in seconds
+    read_timeout: int = 600  # Read timeout in seconds
+    api_key: Optional[str] = None
 
     # Performance and parallelism
     number: Optional[int] = None  # Number of requests to be made
@@ -125,7 +125,13 @@ class ParseKVAction(argparse.Action):
             setattr(namespace, self.dest, {})
         else:
             try:
-                kv_dict = dict(kv.split('=') for kv in values)
+                kv_dict = {}
+                for kv in values:
+                    parts = kv.split('=', 1)  # only split the first '='
+                    if len(parts) != 2:
+                        raise ValueError(f'Invalid key-value pair: {kv}')
+                    key, value = parts
+                    kv_dict[key.strip()] = value.strip()
                 setattr(namespace, self.dest, kv_dict)
             except ValueError as e:
                 parser.error(f'Error parsing key-value pairs: {e}')
@@ -144,9 +150,9 @@ def add_argument(parser: argparse.ArgumentParser):
     parser.add_argument('--url', type=str, default='http://127.0.0.1:8877/v1/chat/completions')
     parser.add_argument('--port', type=int, default=8877, help='The port for local inference')
     parser.add_argument('--headers', nargs='+', dest='headers', action=ParseKVAction, help='Extra HTTP headers')
-    parser.add_argument('--api-key', type=str, required=False, default='EMPTY', help='The API key for authentication')
-    parser.add_argument('--connect-timeout', type=int, default=120, help='The network connection timeout')
-    parser.add_argument('--read-timeout', type=int, default=120, help='The network read timeout')
+    parser.add_argument('--api-key', type=str, required=False, default=None, help='The API key for authentication')
+    parser.add_argument('--connect-timeout', type=int, default=600, help='The network connection timeout')
+    parser.add_argument('--read-timeout', type=int, default=600, help='The network read timeout')
 
     # Performance and parallelism
     parser.add_argument('-n', '--number', type=int, default=None, help='How many requests to be made')

evalscope/perf/benchmark.py

@@ -209,9 +209,14 @@ async def benchmark(args: Arguments) -> None:
         loop = asyncio.get_running_loop()
         add_signal_handlers(loop)
 
+    # init queue
     request_queue = asyncio.Queue()
     benchmark_data_queue = asyncio.Queue()
 
+    # reset event
+    query_send_completed_event.clear()
+    data_process_completed_event.clear()
+
     async def create_send_request_tasks():
         tasks: List[asyncio.Task] = []
         for idx in range(args.parallel):

evalscope/perf/http_client.py

@@ -23,10 +23,7 @@ class AioHttpClient:
         self.read_timeout = args.read_timeout
         self.connect_timeout = args.connect_timeout
         self.client = aiohttp.ClientSession(
-            timeout=aiohttp.ClientTimeout(
-                total=self.read_timeout + self.connect_timeout,
-                connect=self.connect_timeout,
-                sock_read=self.read_timeout),
+            timeout=aiohttp.ClientTimeout(connect=self.connect_timeout, sock_read=self.read_timeout),
             connector=aiohttp.TCPConnector(limit=1),
             trace_configs=[self._create_trace_config()] if args.debug else [])
 
@@ -102,6 +99,11 @@ class AioHttpClient:
             async with self.client.request('POST', url=self.url, data=data, headers=headers) as response:
                 async for rsp in self._handle_response(response):
                     yield rsp
+        except asyncio.TimeoutError:
+            logger.error(
+                f'TimeoutError: connect_timeout: {self.connect_timeout}, read_timeout: {self.read_timeout}. Please set longger timeout.'  # noqa: E501
+            )
+            yield (True, None, 'Timeout')
         except (aiohttp.ClientConnectorError, Exception) as e:
             logger.error(e)
             yield (True, None, e)
@@ -143,7 +145,15 @@ async def test_connection(args: Arguments) -> bool:
         client = AioHttpClient(args)
         async with client:
             if 'chat/completions' in args.url:
-                request = {'messages': [{'role': 'user', 'content': 'hello'}], 'model': args.model, 'max_tokens': 10}
+                request = {
+                    'messages': [{
+                        'role': 'user',
+                        'content': 'hello'
+                    }],
+                    'model': args.model,
+                    'max_tokens': 10,
+                    'stream': args.stream
+                }
             else:
                 request = {'prompt': 'hello', 'model': args.model, 'max_tokens': 10}
             async for is_error, state_code, response_data in client.post(request):

evalscope/perf/main.py

@@ -35,6 +35,7 @@ def run_perf_benchmark(args):
     loop = asyncio.get_event_loop()
     if platform.system() != 'Windows':
         add_signal_handlers(loop)
+
     loop.run_until_complete(benchmark(args))
 
 

evalscope/perf/utils/analysis_result.py

@@ -3,7 +3,7 @@ import json
 import pickle
 import sqlite3
 
-result_db_path = '/mnt/data/data/user/maoyunlin.myl/eval-scope/outputs/qwen2.5_benchmark_20241111_160543.db'
+result_db_path = './outputs/qwen2.5_benchmark_20241111_160543.db'
 con = sqlite3.connect(result_db_path)
 query_sql = "SELECT request, response_messages, prompt_tokens, completion_tokens \
                 FROM result WHERE success='1'"

evalscope/report/app.py

@@ -125,6 +125,9 @@ def get_compare_report_df(acc_df: pd.DataFrame):
 
 
 def plot_single_report_scores(df: pd.DataFrame):
+    if df is None:
+        return None
+    logger.debug(f'df: {df}')
     plot = px.bar(df, x=df[ReportKey.dataset_name], y=df[ReportKey.score], text=df[ReportKey.score])
 
     width = 0.2 if len(df[ReportKey.dataset_name]) <= 5 else None

evalscope/report/combinator.py

@@ -57,8 +57,8 @@ class ReportsRecorder:
 
 
 if __name__ == '__main__':
-    report_dir_1 = '/mnt/data/data/user/maoyunlin.myl/eval-scope/outputs/20250117_151926'
-    # report_dir_2 = '/mnt/data/data/user/maoyunlin.myl/eval-scope/outputs/20250107_204445/reports'
+    report_dir_1 = './outputs/20250117_151926'
+    # report_dir_2 = './outputs/20250107_204445/reports'
 
     report_table = gen_table([report_dir_1])
     print(report_table)

evalscope/run.py

@@ -2,7 +2,7 @@
 """
 Run evaluation for LLMs.
 """
-import os.path
+import os
 from argparse import Namespace
 from datetime import datetime
 from typing import TYPE_CHECKING, List, Optional, Union
@@ -127,16 +127,17 @@ def create_evaluator(task_cfg: TaskConfig, dataset_name: str, outputs: OutputsSt
     from evalscope.models import initialize_model_adapter
 
     benchmark: BenchmarkMeta = Benchmark.get(dataset_name)
-    # Initialize data adapter
-    data_adapter = benchmark.get_data_adapter(config=task_cfg.dataset_args.get(dataset_name, {}))
 
     if dataset_name == DataCollection.NAME:
         # EvaluatorCollection is a collection of evaluators
         from evalscope.collections import EvaluatorCollection
-        return EvaluatorCollection(task_cfg, data_adapter, outputs)
+        data_adapter = benchmark.get_data_adapter(config=task_cfg.dataset_args.get(dataset_name, {}))
+        return EvaluatorCollection(task_cfg, data_adapter, outputs, base_model)
 
     # Initialize model adapter
-    model_adapter = initialize_model_adapter(task_cfg, benchmark.model_adapter, base_model)
+    model_adapter = initialize_model_adapter(task_cfg, benchmark, base_model)
+    # Initialize data adapter
+    data_adapter = benchmark.get_data_adapter(config=task_cfg.dataset_args.get(dataset_name, {}))
 
     # update task_cfg.dataset_args
     task_cfg.dataset_args[dataset_name] = benchmark.to_string_dict()

evalscope/third_party/longbench_write/infer.py

@@ -8,7 +8,7 @@ import random
 import torch
 from typing import List
 
-from evalscope.models.api import OpenaiApi
+from evalscope.third_party.longbench_write.tools.openai_api import OpenaiApi
 from evalscope.third_party.longbench_write.utils import count_words
 from evalscope.utils import get_logger