evalscope 0.17.0__py3-none-any.whl → 0.17.1__py3-none-any.whl

evalscope/models/adapters/base_adapter.py

@@ -53,7 +53,10 @@ def initialize_model_adapter(task_cfg: 'TaskConfig', benchmark: 'DataAdapter', b
         if task_cfg.eval_type == EvalType.SERVICE or task_cfg.api_url is not None:
 
             if 'server' not in model_adapter_cls_str:
+                logger.warning(f'Output type {model_adapter_cls_str} is not supported for service evaluation. '
+                               f'Using server model adapter instead.')
                 model_adapter_cls_str = 'server'
+                benchmark.model_adapter = model_adapter_cls_str
 
             # init server model adapter
             model_adapter_cls = get_model_adapter(model_adapter_cls_str)
@@ -71,6 +74,7 @@ def initialize_model_adapter(task_cfg: 'TaskConfig', benchmark: 'DataAdapter', b
                 logger.warning(f'Output type {model_adapter_cls_str} is not supported for benchmark {benchmark.name}.'
                                f'Using {benchmark.output_types[0]} instead.')
                 model_adapter_cls_str = benchmark.output_types[0]
+                benchmark.model_adapter = model_adapter_cls_str
 
             model_adapter_cls = get_model_adapter(model_adapter_cls_str)
             return model_adapter_cls(

evalscope/models/adapters/bfcl_adapter.py

@@ -4,11 +4,13 @@ import uuid
 from typing import Any, List, Optional, Union
 
 from evalscope.utils.logger import get_logger
+from ..register import register_model_adapter
 from .server_adapter import ServerModelAdapter
 
 logger = get_logger()
 
 
+@register_model_adapter(name='bfcl_server')
 class BFCLAdapter(ServerModelAdapter):
     """
     BFCL model adapter to request remote API model and generate results for BFCL evaluation.

evalscope/models/adapters/chat_adapter.py

@@ -3,15 +3,18 @@ import time
 import torch
 from typing import Any, Dict, List, Optional, Tuple, Union
 
+from evalscope.constants import OutputType
 from evalscope.utils.chat_service import ChatCompletionResponse, ChatCompletionResponseChoice, ChatMessage, Usage
 from evalscope.utils.logger import get_logger
 from evalscope.utils.model_utils import fix_do_sample_warning
 from ..local_model import LocalModel
+from ..register import register_model_adapter
 from .base_adapter import BaseModelAdapter
 
 logger = get_logger()
 
 
+@register_model_adapter(name=OutputType.GENERATION)
 class ChatGenerationModelAdapter(BaseModelAdapter):
     """
     Chat generation model adapter.

evalscope/models/adapters/choice_adapter.py

@@ -3,11 +3,14 @@ import time
 import torch
 from typing import List
 
+from evalscope.constants import OutputType
 from evalscope.utils.chat_service import ChatCompletionResponse, ChatCompletionResponseChoice, ChatMessage
 from ..local_model import LocalModel
+from ..register import register_model_adapter
 from .base_adapter import BaseModelAdapter
 
 
+@register_model_adapter(name=OutputType.MULTIPLE_CHOICE)
 class MultiChoiceModelAdapter(BaseModelAdapter):
     """ The multi-choice model adapter. """
 
@@ -110,6 +113,7 @@ class MultiChoiceModelAdapter(BaseModelAdapter):
         return log_probs, {'tokens': tokens}
 
 
+@register_model_adapter(name=OutputType.CONTINUOUS)
 class ContinuationLogitsModelAdapter(MultiChoiceModelAdapter):
     """
     Continuation-logits model adapter.

evalscope/models/adapters/custom_adapter.py

@@ -1,12 +1,16 @@
-from typing import Any, Dict, List, Union
+from typing import TYPE_CHECKING, Any, Dict, List, Union
 
-from ..custom import CustomModel
+from ..register import register_model_adapter
 from .base_adapter import BaseModelAdapter
 
+if TYPE_CHECKING:
+    from ..custom import CustomModel
 
+
+@register_model_adapter(name='custom')
 class CustomModelAdapter(BaseModelAdapter):
 
-    def __init__(self, custom_model: CustomModel, **kwargs):
+    def __init__(self, custom_model: 'CustomModel', **kwargs):
         """
         Custom model adapter.
 

evalscope/models/adapters/server_adapter.py

@@ -7,11 +7,13 @@ from typing import List, Optional, Union
 
 from evalscope.utils.argument_utils import get_supported_params
 from evalscope.utils.logger import get_logger
+from ..register import register_model_adapter
 from .base_adapter import BaseModelAdapter
 
 logger = get_logger()
 
 
+@register_model_adapter(name='server')
 class ServerModelAdapter(BaseModelAdapter):
     """
     Server model adapter to request remote API model and generate results.

evalscope/models/adapters/t2i_adapter.py

@@ -3,15 +3,18 @@ import time
 import torch
 from typing import Any, Dict, List, Optional, Tuple, Union
 
+from evalscope.constants import OutputType
 from evalscope.utils.chat_service import ChatCompletionResponse, ChatCompletionResponseChoice, ChatMessage
 from evalscope.utils.io_utils import OutputsStructure
 from evalscope.utils.logger import get_logger
 from ..local_model import LocalModel
+from ..register import register_model_adapter
 from .base_adapter import BaseModelAdapter
 
 logger = get_logger()
 
 
+@register_model_adapter(name=OutputType.IMAGE_GENERATION)
 class T2IModelAdapter(BaseModelAdapter):
     """
     Text to image model adapter.

evalscope/models/adapters/tau_bench_adapter.py

@@ -0,0 +1,189 @@
+import json
+import time
+from typing import Any, Dict, List, Optional, Union
+
+from evalscope.utils.logger import get_logger
+from ..register import register_model_adapter
+from .server_adapter import ServerModelAdapter
+
+logger = get_logger()
+
+
+@register_model_adapter(name='tau_bench_server')
+class TauBenchAdapter(ServerModelAdapter):
+    """
+    TauBench model adapter to request remote API model and generate results for TauBench evaluation.
+    Support multi-turn and single-turn function calling tasks.
+    """
+
+    def __init__(self, api_url: str, model_id: str, api_key: str = 'EMPTY', **kwargs):
+        """
+        Args:
+            api_url: The URL of the remote API model.
+            model_id: The ID of the remote API model.
+            api_key: The API key of the remote API model.
+        """
+        super().__init__(api_url=api_url, model_id=model_id, api_key=api_key, **kwargs)
+
+        self._patch_agent_solve()
+
+    def predict(self, inputs: List[dict], infer_cfg: Optional[dict] = None) -> List[dict]:
+        """
+        Model prediction func. For multi-turn evals, we pass a list[list[message]] to the model
+        where each list is a follow up turn in the conversation
+        each turn is a List[List[Message]]
+
+        Args:
+            inputs (List[dict]): The input data.
+            infer_cfg (dict): Inference configuration.
+
+        Returns:
+            res (List[dict]): The model prediction results.
+        """
+        infer_cfg = infer_cfg or {}
+        results = []
+
+        for input_item in inputs:
+            raw_input = input_item.get('raw_input')
+
+            res_d = self.solve(env_name=raw_input['env_name'], task_index=raw_input['task_index'], infer_cfg=infer_cfg)
+
+            wrapper_res = {
+                'choices': [{
+                    'index': 0,
+                    'message': {
+                        'content': json.dumps(res_d, ensure_ascii=False),
+                        'role': 'assistant'
+                    }
+                }],
+                'created':
+                time.time(),
+                'model':
+                self.model_id,
+                'object':
+                'chat.completion',
+                'usage': {
+                    'completion_tokens': 0,
+                    'prompt_tokens': 0,
+                    'total_tokens': 0
+                }
+            }
+
+            results.append(wrapper_res)
+
+        return results
+
+    def _patch_agent_solve(self):
+        """Patch ToolCallingAgent.solve method to use custom model configuration"""
+        from tau_bench.agents.tool_calling_agent import ToolCallingAgent, message_to_action
+        from tau_bench.envs.base import Env
+        from tau_bench.types import RESPOND_ACTION_NAME, SolveResult
+        from typing import List, Optional
+
+        def patched_solve(self,
+                          env: Env,
+                          task_index: Optional[int] = None,
+                          max_num_steps: int = 30,
+                          infer_cfg: Optional[dict] = {}) -> SolveResult:
+            env_reset_res = env.reset(task_index=task_index)
+            obs = env_reset_res.observation
+            info = env_reset_res.info.model_dump()
+            reward = 0.0
+            messages: List[Dict[str, Any]] = [
+                {
+                    'role': 'system',
+                    'content': self.wiki
+                },
+                {
+                    'role': 'user',
+                    'content': obs
+                },
+            ]
+
+            for step_index in range(max_num_steps):
+                # Use adapter's model configuration instead of agent's
+                request_json = adapter_instance.make_request(
+                    input_item={
+                        'messages': messages,
+                        'tools': self.tools_info
+                    }, infer_cfg=infer_cfg)
+                res = adapter_instance.send_request(request_json)
+
+                next_message = res['choices'][0]['message']
+                action = message_to_action(next_message)
+                env_response = env.step(action)
+                reward = env_response.reward
+                info = {**info, **env_response.info.model_dump()}
+
+                if action.name != RESPOND_ACTION_NAME:
+                    next_message['tool_calls'] = next_message['tool_calls'][:1]
+                    messages.extend([
+                        next_message,
+                        {
+                            'role': 'tool',
+                            'tool_call_id': next_message['tool_calls'][0]['id'],
+                            'name': next_message['tool_calls'][0]['function']['name'],
+                            'content': env_response.observation,
+                        },
+                    ])
+                else:
+                    messages.extend([
+                        next_message,
+                        {
+                            'role': 'user',
+                            'content': env_response.observation
+                        },
+                    ])
+                logger.debug(f'Task: {task_index} Step: {step_index} finished')
+
+                if env_response.done:
+                    break
+
+            return SolveResult(
+                reward=reward,
+                info=info,
+                messages=messages,
+                total_cost=0,
+            )
+
+        adapter_instance = self
+
+        ToolCallingAgent.solve = patched_solve
+
+        return 'ToolCallingAgent.solve patched successfully'
+
+    def solve(self, env_name, task_index, infer_cfg, **kwargs):
+        """
+        Solve a specific task in the TauBench environment.
+
+        Args:
+            env_name (str): The name of the TauBench environment.
+            task_index (int): The index of the task to solve.
+            **kwargs: Additional arguments for the task.
+
+        Returns:
+            dict: The result of the task.
+        """
+        from tau_bench.agents.tool_calling_agent import ToolCallingAgent
+        from tau_bench.envs import get_env
+
+        # This method can be implemented to solve specific tasks in the TauBench environment
+        isolated_env = get_env(
+            env_name=env_name,
+            user_strategy='llm',
+            user_model='dummy',  # Use dummy model to prevent errors
+            user_provider='openai',  # Use dummy provider to prevent errors
+            task_split='test',
+            task_index=task_index,
+        )
+        agent = ToolCallingAgent(
+            tools_info=isolated_env.tools_info,
+            wiki=isolated_env.wiki,
+            model='dummy',  # Use dummy model to prevent errors
+            provider='dummy',  # Use dummy provider to prevent errors
+            temperature=0,  # dummy temperature to prevent errors
+        )
+
+        res = agent.solve(env=isolated_env, task_index=task_index, infer_cfg=infer_cfg)
+
+        return res.model_dump()

evalscope/models/register.py

@@ -1,6 +1,3 @@
-from evalscope.constants import OutputType
-from .adapters import *
-
 MODEL_ADAPTERS = {}
 
 
@@ -42,14 +39,3 @@ def register_model_adapter_class(cls, name=None):
     if name in MODEL_ADAPTERS:
         raise ValueError(f"Model adapter class '{name}' is already registered.")
     MODEL_ADAPTERS[name] = cls
-
-
-# register all model adapters
-register_model_adapter_class(BaseModelAdapter, name='base')
-register_model_adapter_class(ChatGenerationModelAdapter, name=OutputType.GENERATION)
-register_model_adapter_class(ContinuationLogitsModelAdapter, name=OutputType.CONTINUOUS)
-register_model_adapter_class(MultiChoiceModelAdapter, name=OutputType.MULTIPLE_CHOICE)
-register_model_adapter_class(CustomModelAdapter, name='custom')
-register_model_adapter_class(ServerModelAdapter, name='server')
-register_model_adapter_class(BFCLAdapter, name='bfcl_server')
-register_model_adapter_class(T2IModelAdapter, name=OutputType.IMAGE_GENERATION)

evalscope/perf/arguments.py

@@ -31,6 +31,7 @@ class Arguments(BaseArgument):
     number: Union[int, List[int]] = 1000  # Number of requests to be made
     parallel: Union[int, List[int]] = 1  # Number of parallel requests
     rate: int = -1  # Rate limit for requests (default: -1, no limit)
+    sleep_interval: int = 5  # Sleep interval between performance runs, in seconds
 
     # Logging and debugging
     log_every_n_query: int = 10  # Log every N queries
@@ -49,6 +50,11 @@ class Arguments(BaseArgument):
     prompt: Optional[str] = None  # The prompt text
     query_template: Optional[str] = None  # Template for the query
     apply_chat_template: Optional[bool] = None  # Whether to apply chat template
+    # random vl settings
+    image_width: int = 224  # Width of the image for random VL dataset
+    image_height: int = 224  # Height of the image for random VL dataset
+    image_format: str = 'RGB'  # Image format for random VL dataset
+    image_num: int = 1  # Number of images for random VL dataset
 
     # Dataset settings
     dataset: str = 'openqa'  # Dataset type (default: 'line_by_line')
@@ -142,6 +148,8 @@ def add_argument(parser: argparse.ArgumentParser):
     parser.add_argument('-n', '--number', type=int, default=1000, nargs='+', help='How many requests to be made')
     parser.add_argument('--parallel', type=int, default=1, nargs='+', help='Set number of concurrency requests, default 1')  # noqa: E501
     parser.add_argument('--rate', type=int, default=-1, help='Number of requests per second. default None')
+    parser.add_argument(
+        '--sleep-interval', type=int, default=5, help='Sleep interval between performance runs, in seconds. Default 5')  # noqa: E501
 
     # Logging and debugging
     parser.add_argument('--log-every-n-query', type=int, default=10, help='Logging every n query')
@@ -158,6 +166,11 @@ def add_argument(parser: argparse.ArgumentParser):
     parser.add_argument('--query-template', type=str, default=None, help='Specify the query template')
     parser.add_argument(
         '--apply-chat-template', type=argparse.BooleanOptionalAction, default=None, help='Apply chat template to the prompt')  # noqa: E501
+    # random vl settings
+    parser.add_argument('--image-width', type=int, default=224, help='Width of the image for random VL dataset')
+    parser.add_argument('--image-height', type=int, default=224, help='Height of the image for random VL dataset')
+    parser.add_argument('--image-format', type=str, default='RGB', help='Image format for random VL dataset')
+    parser.add_argument('--image-num', type=int, default=1, help='Number of images for random VL dataset')
 
     # Output settings
     parser.add_argument('--outputs-dir', help='Outputs dir.', default='outputs')

evalscope/perf/benchmark.py

@@ -6,15 +6,18 @@ import sqlite3
 import time
 from http import HTTPStatus
 from tqdm import tqdm
-from typing import AsyncGenerator, Dict, List, Tuple
-
-from evalscope.perf.arguments import Arguments
-from evalscope.perf.http_client import AioHttpClient, test_connection
-from evalscope.perf.plugin.registry import ApiRegistry, DatasetRegistry
-from evalscope.perf.utils.benchmark_util import BenchmarkData, BenchmarkMetrics
-from evalscope.perf.utils.db_util import create_result_table, get_result_db_path, insert_benchmark_data, summary_result
-from evalscope.perf.utils.handler import add_signal_handlers, exception_handler
+from typing import TYPE_CHECKING, AsyncGenerator, Dict, List, Tuple
+
 from evalscope.utils.logger import get_logger
+from .arguments import Arguments
+from .http_client import AioHttpClient, test_connection
+from .plugin import ApiRegistry, DatasetRegistry
+from .utils.benchmark_util import BenchmarkData, BenchmarkMetrics
+from .utils.db_util import create_result_table, get_result_db_path, insert_benchmark_data, load_prompt, summary_result
+from .utils.handler import add_signal_handlers, exception_handler
+
+if TYPE_CHECKING:
+    from .plugin import ApiPluginBase, DatasetPluginBase
 
 logger = get_logger()
 
@@ -22,28 +25,22 @@ data_process_completed_event = asyncio.Event()
 
 
 @exception_handler
-async def get_requests(args: Arguments) -> AsyncGenerator[dict, None]:
-    query_generator_class = ApiRegistry(args.api)
-    query_generator = query_generator_class(args.tokenizer_path)
-
-    def load_prompt(prompt_path_or_text):
-        if prompt_path_or_text.startswith('@'):
-            with open(prompt_path_or_text[1:], 'r', encoding='utf-8') as file:
-                return file.read()
-        return prompt_path_or_text
-
-    async def generate_requests_from_prompt(messages):
-        request = query_generator.build_request(messages, args)
+async def get_requests(args: Arguments, api_plugin: 'ApiPluginBase') -> AsyncGenerator[dict, None]:
+
+    async def generate_requests_from_prompt():
+        prompt = load_prompt(args.prompt)
+        messages = [{'role': 'user', 'content': prompt}] if args.apply_chat_template else prompt
+        request = api_plugin.build_request(messages)
         for _ in range(args.number):
             yield request
 
     async def generate_requests_from_dataset():
-        message_generator_class = DatasetRegistry(args.dataset)
+        message_generator_class = DatasetRegistry.get_class(args.dataset)
         message_generator = message_generator_class(args)
 
         dataset_messages = []
         try:
-            for messages in message_generator:
+            for messages in message_generator.build_messages():
                 dataset_messages.append(messages)
         except StopIteration:
             pass
@@ -56,7 +53,7 @@ async def get_requests(args: Arguments) -> AsyncGenerator[dict, None]:
 
         while count < args.number:
             messages = dataset_messages[dataset_index]
-            request = query_generator.build_request(messages, args)
+            request = api_plugin.build_request(messages)
             if request is not None:
                 yield request
                 count += 1
@@ -64,13 +61,11 @@ async def get_requests(args: Arguments) -> AsyncGenerator[dict, None]:
             dataset_index = (dataset_index + 1) % len(dataset_messages)
 
     if args.prompt:
-        prompt = load_prompt(args.prompt)
-        messages = [{'role': 'user', 'content': prompt}] if args.apply_chat_template else prompt
-        generator = generate_requests_from_prompt(messages)
+        generator = generate_requests_from_prompt()
     elif args.dataset:
         generator = generate_requests_from_dataset()
     else:
-        raise Exception('Either prompt or dataset is required!')
+        raise ValueError('Either prompt or dataset is required!')
 
     async for request in generator:
         yield request
@@ -85,9 +80,10 @@ async def send_request(
     request: dict,
     benchmark_data_queue: asyncio.Queue,
     args: Arguments,
+    api_plugin: 'ApiPluginBase',
 ):
     async with semaphore:
-        client = AioHttpClient(args)
+        client = AioHttpClient(args, api_plugin)
         async with client:
             benchmark_data = BenchmarkData(request=request)
             benchmark_data.start_time = time.perf_counter()
@@ -95,7 +91,8 @@ async def send_request(
             try:
                 async for is_error, state_code, response_data in client.post(request):
                     if is_error or state_code != HTTPStatus.OK:
-                        logger.error(f'Request: {request} failed, state_code: {state_code}, data: {response_data}')
+                        error_msg = str(response_data) if response_data else 'Unknown error'
+                        logger.error(f'Request: {request} failed, state_code: {state_code}, data: {error_msg}')
                         benchmark_data.success = False
                         break
                     if response_data:
@@ -116,12 +113,9 @@ async def send_request(
 
 
 @exception_handler
-async def statistic_benchmark_metric(benchmark_data_queue: asyncio.Queue, args: Arguments):
+async def statistic_benchmark_metric(benchmark_data_queue: asyncio.Queue, args: Arguments, api_plugin: 'ApiPluginBase'):
     metrics = BenchmarkMetrics(concurrency=args.parallel)
 
-    api_plugin_class = ApiRegistry(args.api)
-    api_plugin = api_plugin_class(args.tokenizer_path)
-
     result_db_path = get_result_db_path(args)
 
     collected_benchmark_data = []
@@ -172,8 +166,8 @@ async def statistic_benchmark_metric(benchmark_data_queue: asyncio.Queue, args:
 
 
 @exception_handler
-async def connect_test(args: Arguments) -> bool:
-    if (not args.no_test_connection) and (not await test_connection(args)):
+async def connect_test(args: Arguments, api_plugin) -> bool:
+    if (not args.no_test_connection) and (not await test_connection(args, api_plugin)):
         raise TimeoutError('Test connection failed')
 
 
@@ -183,19 +177,24 @@ async def benchmark(args: Arguments) -> Tuple[Dict, Dict]:
         loop = asyncio.get_running_loop()
         add_signal_handlers(loop)
 
+    # Create API plugin instance for request/response processing
+    api_plugin_class = ApiRegistry.get_class(args.api)
+    api_plugin = api_plugin_class(args)
+
     # init queue
     benchmark_data_queue = asyncio.Queue()
     # reset event
     data_process_completed_event.clear()
     # test connection
-    await connect_test(args)
+    await connect_test(args, api_plugin)
     # start statistic benchmark metric
-    statistic_benchmark_metric_task = asyncio.create_task(statistic_benchmark_metric(benchmark_data_queue, args))
+    statistic_benchmark_metric_task = asyncio.create_task(
+        statistic_benchmark_metric(benchmark_data_queue, args, api_plugin))
     # start send request
     semaphore = asyncio.Semaphore(args.parallel)
     send_request_tasks: List[asyncio.Task] = []
-    async for request in get_requests(args):
-        task = asyncio.create_task(send_request(semaphore, request, benchmark_data_queue, args))
+    async for request in get_requests(args, api_plugin):
+        task = asyncio.create_task(send_request(semaphore, request, benchmark_data_queue, args, api_plugin))
         send_request_tasks.append(task)
 
     await asyncio.gather(*send_request_tasks, return_exceptions=True)