evalscope 0.17.1__py3-none-any.whl → 1.0.0__py3-none-any.whl

evalscope/models/adapters/server_adapter.py

@@ -1,236 +0,0 @@
-import copy
-import openai
-from collections import defaultdict
-from openai.types.chat import ChatCompletion, ChatCompletionChunk
-from openai.types.chat.chat_completion import ChatCompletionMessage, Choice
-from typing import List, Optional, Union
-
-from evalscope.utils.argument_utils import get_supported_params
-from evalscope.utils.logger import get_logger
-from ..register import register_model_adapter
-from .base_adapter import BaseModelAdapter
-
-logger = get_logger()
-
-
-@register_model_adapter(name='server')
-class ServerModelAdapter(BaseModelAdapter):
-    """
-    Server model adapter to request remote API model and generate results.
-    """
-
-    def __init__(self, api_url: str, model_id: str, api_key: str = 'EMPTY', **kwargs):
-        """
-        Args:
-            api_url: The URL of the remote API model.
-            model_id: The ID of the remote API model.
-            api_key: The API key of the remote API model.
-        """
-        self.api_url = api_url.rstrip('/').rsplit('/chat/completions', 1)[0]
-        self.model_id = model_id
-        self.api_key = api_key
-
-        self.client = openai.OpenAI(
-            api_key=self.api_key,
-            base_url=self.api_url,
-        )
-        self.supported_params = get_supported_params(self.client.chat.completions.create)
-
-        self.seed = kwargs.get('seed', None)
-        self.timeout = kwargs.get('timeout', 60)
-        self.stream = kwargs.get('stream', False)
-        self.model_cfg = {'api_url': api_url, 'model_id': model_id, 'api_key': api_key}
-        super().__init__(model=None, model_cfg=self.model_cfg, **kwargs)
-
-    def predict(self, inputs: List[dict], infer_cfg: Optional[dict] = None) -> List[dict]:
-        """
-        Model prediction func.
-
-        Args:
-            inputs (List[dict]): The input data.
-            infer_cfg (dict): Inference configuration.
-
-        Returns:
-            res (List[dict]): The model prediction results.
-        """
-        infer_cfg = infer_cfg or {}
-        results = []
-
-        for input_item in inputs:
-            response = self.process_single_input(input_item, infer_cfg)
-            results.append(response)
-
-        return results
-
-    def process_single_input(self, input_item: dict, infer_cfg: dict) -> dict:
-        """Process a single input item."""
-        request_json = self.make_request(input_item, infer_cfg)
-        response = self.send_request(request_json)
-        return response
-
-    def make_request_messages(self, input_item: dict) -> list:
-        """
-        Make request messages for OpenAI API.
-        """
-        if input_item.get('messages', None):
-            return input_item['messages']
-
-        data: list = input_item['data']
-        if isinstance(data[0], tuple):  # for truthful_qa and hellaswag
-            query = '\n'.join(''.join(item) for item in data)
-            system_prompt = input_item.get('system_prompt', None)
-        else:
-            query = data[0]
-            system_prompt = input_item.get('system_prompt', None)
-
-        messages = []
-        if system_prompt:
-            messages.append({'role': 'system', 'content': system_prompt})
-
-        messages.append({'role': 'user', 'content': query})
-
-        return messages
-
-    def make_request(self, input_item: dict, infer_cfg: dict) -> dict:
-        """Make request to remote API."""
-        messages = self.make_request_messages(input_item)
-        # Format request JSON according to OpenAI API format
-        request_json = {'model': self.model_id, 'messages': messages, **infer_cfg}
-
-        if self.timeout:
-            request_json['timeout'] = self.timeout
-
-        request_json['stream'] = self.stream
-        if self.stream:
-            request_json['stream_options'] = {'include_usage': True}
-
-        if input_item.get('tools', None):
-            tools_copy = copy.deepcopy(input_item.get('tools'))
-            # Remove the "responses" from the functions, as that doesn't
-            # need to go to the model
-            for tool in tools_copy:
-                if 'function' in tool and 'response' in tool['function']:
-                    del tool['function']['response']
-            request_json['tools'] = tools_copy
-
-        logger.debug(f'Request to remote API: {request_json}')
-
-        return request_json
-
-    def _parse_extra_params(self, request_json):
-        api_params = {}
-        extra_body = {}
-        for key, value in request_json.items():
-            if key in self.supported_params:
-                api_params[key] = value
-            else:
-                extra_body[key] = value
-
-        if extra_body:
-            api_params['extra_body'] = extra_body
-        return api_params
-
-    def send_request(self, request_json: dict) -> dict:
-        try:
-            parsed_request = self._parse_extra_params(request_json)
-            response = self.client.chat.completions.create(**parsed_request)
-
-            if response and self.stream:
-                response = self._collect_stream_response(response)
-
-            return response.model_dump(exclude_unset=True)
-        except Exception as e:
-            logger.error(f'Error when calling remote API: {str(e)}')
-            raise e
-
-    def _collect_stream_response(self, response_stream: List[ChatCompletionChunk]) -> ChatCompletion:
-        collected_chunks = []
-        collected_messages = defaultdict(list)
-        collected_reasoning = defaultdict(list)
-        collected_tool_calls = defaultdict(dict)
-
-        for chunk in response_stream:
-            collected_chunks.append(chunk)
-            for choice in chunk.choices:
-                # Handle reasoning content
-                if hasattr(choice.delta, 'reasoning_content') and choice.delta.reasoning_content is not None:
-                    collected_reasoning[choice.index].append(choice.delta.reasoning_content)
-
-                # Handle regular content
-                if choice.delta.content is not None:
-                    collected_messages[choice.index].append(choice.delta.content)
-
-                # Handle tool calls
-                if hasattr(choice.delta, 'tool_calls') and choice.delta.tool_calls:
-                    for tool_call in choice.delta.tool_calls:
-                        tool_id = tool_call.index
-
-                        # Initialize tool call if not present
-                        if tool_id not in collected_tool_calls[choice.index]:
-                            collected_tool_calls[choice.index][tool_id] = {
-                                'id': tool_call.id if hasattr(tool_call, 'id') and tool_call.id else None,
-                                'type': tool_call.type if hasattr(tool_call, 'type') and tool_call.type else None,
-                                'function': {
-                                    'name': '',
-                                    'arguments': ''
-                                }
-                            }
-
-                        # Update tool call with new chunks
-                        if hasattr(tool_call, 'function'):
-                            if hasattr(tool_call.function, 'name') and tool_call.function.name:
-                                collected_tool_calls[
-                                    choice.index][tool_id]['function']['name'] = tool_call.function.name
-
-                            if hasattr(tool_call.function, 'arguments') and tool_call.function.arguments:
-                                collected_tool_calls[
-                                    choice.index][tool_id]['function']['arguments'] += tool_call.function.arguments
-
-                        # Update ID if it was received later
-                        if hasattr(tool_call, 'id') and tool_call.id:
-                            collected_tool_calls[choice.index][tool_id]['id'] = tool_call.id
-
-        # Get all unique choice indices from all collections
-        all_indices = set(collected_messages.keys()) | set(collected_reasoning.keys()) | set(
-            collected_tool_calls.keys())
-
-        choices = []
-        for index in all_indices:
-            full_reply_content = ''.join(collected_messages.get(index, []))
-            reasoning = ''.join(collected_reasoning.get(index, []))
-
-            # Process tool_calls for this choice if any exists
-            tool_calls_list = None
-            if index in collected_tool_calls and collected_tool_calls[index]:
-                tool_calls_list = list(collected_tool_calls[index].values())
-                # Filter out any tool calls with None id (incomplete tool calls)
-                tool_calls_list = [tc for tc in tool_calls_list if tc['id'] is not None]
-
-            # use the finish_reason from the last chunk that generated this choice
-            finish_reason = None
-            for chunk in reversed(collected_chunks):
-                if chunk.choices and chunk.choices[0].index == index:
-                    finish_reason = chunk.choices[0].finish_reason
-                    break
-
-            message_kwargs = {'role': 'assistant', 'content': full_reply_content}
-
-            if reasoning:
-                message_kwargs['reasoning_content'] = reasoning
-
-            if tool_calls_list:
-                message_kwargs['tool_calls'] = tool_calls_list
-
-            choice = Choice(
-                finish_reason=finish_reason or 'stop', index=index, message=ChatCompletionMessage(**message_kwargs))
-            choices.append(choice)
-
-        # build the final completion object
-        return ChatCompletion(
-            id=collected_chunks[0].id,
-            choices=choices,
-            created=collected_chunks[0].created,
-            model=collected_chunks[0].model,
-            object='chat.completion',
-            usage=collected_chunks[-1].usage  # use the usage from the last chunk
-        )

evalscope/models/adapters/t2i_adapter.py

@@ -1,79 +0,0 @@
-import os
-import time
-import torch
-from typing import Any, Dict, List, Optional, Tuple, Union
-
-from evalscope.constants import OutputType
-from evalscope.utils.chat_service import ChatCompletionResponse, ChatCompletionResponseChoice, ChatMessage
-from evalscope.utils.io_utils import OutputsStructure
-from evalscope.utils.logger import get_logger
-from ..local_model import LocalModel
-from ..register import register_model_adapter
-from .base_adapter import BaseModelAdapter
-
-logger = get_logger()
-
-
-@register_model_adapter(name=OutputType.IMAGE_GENERATION)
-class T2IModelAdapter(BaseModelAdapter):
-    """
-    Text to image model adapter.
-    """
-
-    def __init__(self, model: LocalModel, **kwargs):
-        super().__init__(model)
-
-        self.task_config = kwargs.get('task_cfg', None)
-        assert self.task_config is not None, 'Task config is required for T2I model adapter.'
-
-        self.save_path = os.path.join(self.task_config.work_dir, OutputsStructure.PREDICTIONS_DIR,
-                                      self.task_config.model_id, 'images')
-        os.makedirs(self.save_path, exist_ok=True)
-
-    def _model_generate(self, prompt, infer_cfg=None) -> List:
-        """
-        Generate images from the model.
-        Args:
-            prompt: The input prompt.
-            infer_cfg: The inference configuration.
-        Returns:
-            The generated images.
-        """
-        infer_cfg = infer_cfg or {}
-
-        sample = self.model(prompt=prompt, **infer_cfg).images
-        return sample
-
-    @torch.no_grad()
-    def predict(self, inputs: List[dict], infer_cfg: Optional[dict] = None) -> List[dict]:
-        """
-        Args:
-            inputs: The input data.
-            infer_cfg: The inference configuration.
-        Returns:
-            The prediction results.
-        """
-        results = []
-        for input_item in inputs:
-            prompt = input_item['data'][0]
-            image_id = input_item.get('id') or input_item.get('index')
-
-            samples = self._model_generate(prompt, infer_cfg)
-
-            choices_list = []
-            for index, sample in enumerate(samples):
-                image_file_path = os.path.join(self.save_path, f'{image_id}_{index}.jpeg')
-                sample.save(image_file_path)
-                logger.debug(f'Saved image to {image_file_path}')
-
-                choice = ChatCompletionResponseChoice(
-                    index=index, message=ChatMessage(content=image_file_path, role='assistant'), finish_reason='stop')
-                choices_list.append(choice)
-
-            res_d = ChatCompletionResponse(
-                model=self.model_id, choices=choices_list, object='images.generations',
-                created=int(time.time())).model_dump(exclude_unset=True)
-
-            results.append(res_d)
-
-        return results

evalscope/models/adapters/tau_bench_adapter.py

@@ -1,189 +0,0 @@
-import json
-import time
-from typing import Any, Dict, List, Optional, Union
-
-from evalscope.utils.logger import get_logger
-from ..register import register_model_adapter
-from .server_adapter import ServerModelAdapter
-
-logger = get_logger()
-
-
-@register_model_adapter(name='tau_bench_server')
-class TauBenchAdapter(ServerModelAdapter):
-    """
-    TauBench model adapter to request remote API model and generate results for TauBench evaluation.
-    Support multi-turn and single-turn function calling tasks.
-    """
-
-    def __init__(self, api_url: str, model_id: str, api_key: str = 'EMPTY', **kwargs):
-        """
-        Args:
-            api_url: The URL of the remote API model.
-            model_id: The ID of the remote API model.
-            api_key: The API key of the remote API model.
-        """
-        super().__init__(api_url=api_url, model_id=model_id, api_key=api_key, **kwargs)
-
-        self._patch_agent_solve()
-
-    def predict(self, inputs: List[dict], infer_cfg: Optional[dict] = None) -> List[dict]:
-        """
-        Model prediction func. For multi-turn evals, we pass a list[list[message]] to the model
-        where each list is a follow up turn in the conversation
-        each turn is a List[List[Message]]
-
-        Args:
-            inputs (List[dict]): The input data.
-            infer_cfg (dict): Inference configuration.
-
-        Returns:
-            res (List[dict]): The model prediction results.
-        """
-        infer_cfg = infer_cfg or {}
-        results = []
-
-        for input_item in inputs:
-            raw_input = input_item.get('raw_input')
-
-            res_d = self.solve(env_name=raw_input['env_name'], task_index=raw_input['task_index'], infer_cfg=infer_cfg)
-
-            wrapper_res = {
-                'choices': [{
-                    'index': 0,
-                    'message': {
-                        'content': json.dumps(res_d, ensure_ascii=False),
-                        'role': 'assistant'
-                    }
-                }],
-                'created':
-                time.time(),
-                'model':
-                self.model_id,
-                'object':
-                'chat.completion',
-                'usage': {
-                    'completion_tokens': 0,
-                    'prompt_tokens': 0,
-                    'total_tokens': 0
-                }
-            }
-
-            results.append(wrapper_res)
-
-        return results
-
-    def _patch_agent_solve(self):
-        """Patch ToolCallingAgent.solve method to use custom model configuration"""
-        from tau_bench.agents.tool_calling_agent import ToolCallingAgent, message_to_action
-        from tau_bench.envs.base import Env
-        from tau_bench.types import RESPOND_ACTION_NAME, SolveResult
-        from typing import List, Optional
-
-        def patched_solve(self,
-                          env: Env,
-                          task_index: Optional[int] = None,
-                          max_num_steps: int = 30,
-                          infer_cfg: Optional[dict] = {}) -> SolveResult:
-            env_reset_res = env.reset(task_index=task_index)
-            obs = env_reset_res.observation
-            info = env_reset_res.info.model_dump()
-            reward = 0.0
-            messages: List[Dict[str, Any]] = [
-                {
-                    'role': 'system',
-                    'content': self.wiki
-                },
-                {
-                    'role': 'user',
-                    'content': obs
-                },
-            ]
-
-            for step_index in range(max_num_steps):
-                # Use adapter's model configuration instead of agent's
-                request_json = adapter_instance.make_request(
-                    input_item={
-                        'messages': messages,
-                        'tools': self.tools_info
-                    }, infer_cfg=infer_cfg)
-                res = adapter_instance.send_request(request_json)
-
-                next_message = res['choices'][0]['message']
-                action = message_to_action(next_message)
-                env_response = env.step(action)
-                reward = env_response.reward
-                info = {**info, **env_response.info.model_dump()}
-
-                if action.name != RESPOND_ACTION_NAME:
-                    next_message['tool_calls'] = next_message['tool_calls'][:1]
-                    messages.extend([
-                        next_message,
-                        {
-                            'role': 'tool',
-                            'tool_call_id': next_message['tool_calls'][0]['id'],
-                            'name': next_message['tool_calls'][0]['function']['name'],
-                            'content': env_response.observation,
-                        },
-                    ])
-                else:
-                    messages.extend([
-                        next_message,
-                        {
-                            'role': 'user',
-                            'content': env_response.observation
-                        },
-                    ])
-                logger.debug(f'Task: {task_index} Step: {step_index} finished')
-
-                if env_response.done:
-                    break
-
-            return SolveResult(
-                reward=reward,
-                info=info,
-                messages=messages,
-                total_cost=0,
-            )
-
-        adapter_instance = self
-
-        ToolCallingAgent.solve = patched_solve
-
-        return 'ToolCallingAgent.solve patched successfully'
-
-    def solve(self, env_name, task_index, infer_cfg, **kwargs):
-        """
-        Solve a specific task in the TauBench environment.
-
-        Args:
-            env_name (str): The name of the TauBench environment.
-            task_index (int): The index of the task to solve.
-            **kwargs: Additional arguments for the task.
-
-        Returns:
-            dict: The result of the task.
-        """
-        from tau_bench.agents.tool_calling_agent import ToolCallingAgent
-        from tau_bench.envs import get_env
-
-        # This method can be implemented to solve specific tasks in the TauBench environment
-        isolated_env = get_env(
-            env_name=env_name,
-            user_strategy='llm',
-            user_model='dummy',  # Use dummy model to prevent errors
-            user_provider='openai',  # Use dummy provider to prevent errors
-            task_split='test',
-            task_index=task_index,
-        )
-        agent = ToolCallingAgent(
-            tools_info=isolated_env.tools_info,
-            wiki=isolated_env.wiki,
-            model='dummy',  # Use dummy model to prevent errors
-            provider='dummy',  # Use dummy provider to prevent errors
-            temperature=0,  # dummy temperature to prevent errors
-        )
-
-        res = agent.solve(env=isolated_env, task_index=task_index, infer_cfg=infer_cfg)
-
-        return res.model_dump()

evalscope/models/custom/__init__.py

@@ -1,4 +0,0 @@
-# Copyright (c) Alibaba, Inc. and its affiliates.
-
-from .custom_model import CustomModel
-from .dummy_model import DummyCustomModel

evalscope/models/custom/custom_model.py

@@ -1,50 +0,0 @@
-# Copyright (c) Alibaba, Inc. and its affiliates.
-import torch
-from abc import ABC, abstractmethod
-from typing import Any, Dict, List, Union
-
-
-class CustomModel(ABC):
-
-    def __init__(self, config: dict, **kwargs):
-        self.config = config
-        self.kwargs = kwargs
-
-    @abstractmethod
-    @torch.no_grad()
-    def predict(self, prompts: List[str], **kwargs) -> List[Dict[str, Any]]:
-        """
-        Model prediction function for batch inputs.
-
-        Args:
-            prompts (str): The input batch of prompts to predict.
-
-            **kwargs: kwargs
-
-        Returns:
-            res (dict): The model prediction results (batch). Format:
-            [
-                {
-                  'choices': [
-                    {
-                      'index': 0,
-                      'message': {
-                        'content': 'xxx',
-                        'role': 'assistant'
-                      }
-                    }
-                  ],
-                  'created': 1677664795,
-                  'model': 'gpt-3.5-turbo-0613',   # should be model_id
-                  'object': 'chat.completion',
-                  'usage': {
-                    'completion_tokens': 17,
-                    'prompt_tokens': 57,
-                    'total_tokens': 74
-                  }
-                }
-            ,
-            ...
-            ]
-        """
-        raise NotImplementedError

evalscope/models/custom/dummy_model.py

@@ -1,99 +0,0 @@
-# Copyright (c) Alibaba, Inc. and its affiliates.
-import time
-from typing import List
-
-from evalscope.models import CustomModel
-from evalscope.utils.logger import get_logger
-
-logger = get_logger()
-
-
-class DummyCustomModel(CustomModel):
-
-    def __init__(self, config: dict = {}, **kwargs):
-        super(DummyCustomModel, self).__init__(config=config, **kwargs)
-
-    def make_request_messages(self, input_item: dict) -> list:
-        """
-        Make request messages for OpenAI API.
-        """
-        if input_item.get('messages', None):
-            return input_item['messages']
-
-        data: list = input_item['data']
-        if isinstance(data[0], tuple):  # for truthful_qa and hellaswag
-            query = '\n'.join(''.join(item) for item in data)
-            system_prompt = input_item.get('system_prompt', None)
-        else:
-            query = data[0]
-            system_prompt = input_item.get('system_prompt', None)
-
-        messages = []
-        if system_prompt:
-            messages.append({'role': 'system', 'content': system_prompt})
-
-        messages.append({'role': 'user', 'content': query})
-
-        return messages
-
-    def predict(self, prompts: List[dict], **kwargs):
-        original_inputs = kwargs.get('origin_inputs', None)
-        infer_cfg = kwargs.get('infer_cfg', None)
-
-        logger.debug(f'** Prompts: {prompts}')
-        if original_inputs is not None:
-            logger.debug(f'** Original inputs: {original_inputs}')
-        if infer_cfg is not None:
-            logger.debug(f'** Inference config: {infer_cfg}')
-
-        # Simulate a response based on the prompts
-        # Must return a list of dicts with the same format as the OpenAI API.
-        responses = []
-        for input_item in original_inputs:
-            # message = self.make_request_messages(input_item)
-            # response = f'Dummy response for prompt: {message}'
-
-            res_d = {
-                'choices': [{
-                    'index': 0,
-                    'message': {
-                        'content': '*PlaceHolder*',
-                        'role': 'assistant'
-                    }
-                }],
-                'created': time.time(),
-                'model': self.config.get('model_id'),
-                'object': 'chat.completion',
-                'usage': {
-                    'completion_tokens': 0,
-                    'prompt_tokens': 0,
-                    'total_tokens': 0
-                }
-            }
-
-            responses.append(res_d)
-
-        return responses
-
-
-if __name__ == '__main__':
-    from evalscope import TaskConfig, run_task
-
-    dummy_model = DummyCustomModel()
-    task_config = TaskConfig(
-        model=dummy_model,
-        model_id='evalscope-model-dummy',
-        datasets=['gsm8k'],
-        eval_type='custom',  # must be custom for custom model evaluation
-        generation_config={
-            'max_new_tokens': 100,
-            'temperature': 0.0,
-            'top_p': 1.0,
-            'top_k': 50,
-            'repetition_penalty': 1.0
-        },
-        debug=True,
-        limit=5,
-    )
-
-    eval_results = run_task(task_cfg=task_config)