evalscope 0.16.0__py3-none-any.whl → 0.16.2__py3-none-any.whl

evalscope/models/adapters/bfcl_adapter.py

@@ -0,0 +1,244 @@
+import json
+import time
+import uuid
+from typing import Any, List, Optional, Union
+
+from evalscope.utils.logger import get_logger
+from .server_adapter import ServerModelAdapter
+
+logger = get_logger()
+
+
+class BFCLAdapter(ServerModelAdapter):
+    """
+    BFCL model adapter to request remote API model and generate results for BFCL evaluation.
+    Support multi-turn and single-turn function calling tasks.
+    """
+
+    def __init__(self, api_url: str, model_id: str, api_key: str = 'EMPTY', **kwargs):
+        """
+        Args:
+            api_url: The URL of the remote API model.
+            model_id: The ID of the remote API model.
+            api_key: The API key of the remote API model.
+        """
+        super().__init__(api_url=api_url, model_id=model_id, api_key=api_key, **kwargs)
+
+    def predict(self, inputs: List[dict], infer_cfg: Optional[dict] = None) -> List[dict]:
+        """
+        Model prediction func. For multi-turn evals, we pass a list[list[message]] to the model
+        where each list is a follow up turn in the conversation
+        each turn is a List[List[Message]]
+
+        Args:
+            inputs (List[dict]): The input data.
+            infer_cfg (dict): Inference configuration.
+
+        Returns:
+            res (List[dict]): The model prediction results.
+        """
+        infer_cfg = infer_cfg or {}
+        results = []
+
+        for input_item in inputs:
+            # This flag decides if we pass tools to the API or try tool calling via prompting
+            # Passing tools to the API means that we rely on the API to manage system prompt specifics
+            # and also expect parsed tool calls in the ChatCompletionMessage object
+            # This is how the is_fc_model=True benchmark is designed to work
+            # On the other hand, we try to manage
+            # tool calling via prompting and parse tool calls in the standard text response
+            # This is how the is_fc_model=False benchmark is designed to work
+            row = input_item.get('messages')
+            is_fc_model = row.get('is_fc_model', False)
+
+            if is_fc_model:
+                response = self.generate_turn_with_tools(row, infer_cfg)
+            else:
+                response = self.generate_turn(row, infer_cfg)
+
+            # wrap response with openai types
+            res_d = {
+                'choices': [{
+                    'index': 0,
+                    'message': {
+                        'content': response,
+                        'role': 'assistant'
+                    }
+                }],
+                'created': time.time(),
+                'model': self.model_id,
+                'object': 'chat.completion',
+                'usage': {
+                    'completion_tokens': 0,
+                    'prompt_tokens': 0,
+                    'total_tokens': 0
+                }
+            }
+            results.append(res_d)
+
+        return results
+
+    def generate_turn(self, row: dict[str, Any], infer_cfg: dict[str, Any]) -> list[str]:
+        from bfcl_eval.constants.default_prompts import (DEFAULT_USER_PROMPT_FOR_ADDITIONAL_FUNCTION_PROMPTING,
+                                                         MAXIMUM_STEP_LIMIT)
+        from bfcl_eval.eval_checker.multi_turn_eval.multi_turn_utils import execute_multi_turn_func_call
+        from bfcl_eval.model_handler.utils import default_decode_execute_prompting
+
+        all_model_responses = []
+        current_messages = []
+        turns = row['turns']
+        for turn_idx, messages in enumerate(turns):
+            n_steps = 0
+            current_responses = []
+            current_messages += messages.copy()
+
+            if str(turn_idx) in row['missing_functions']:
+                assert len(messages) == 0, 'Holdout turn should not have user message.'
+                new_turn = [{
+                    'role':
+                    'user',
+                    'content':
+                    DEFAULT_USER_PROMPT_FOR_ADDITIONAL_FUNCTION_PROMPTING.format(
+                        functions=row['missing_functions'][str(turn_idx)]),
+                }]
+                current_messages += new_turn
+
+            while True:
+                input_item = {
+                    'messages': current_messages,
+                }
+                responses = self.process_single_input(input_item, infer_cfg)
+                result = responses['choices'][0]['message']['content']
+
+                logger.debug(f'Turn:{turn_idx} Step:{n_steps} Result: {result}')
+                current_messages.append({
+                    'role': 'assistant',
+                    'content': result,
+                })
+                current_responses.append(result)
+
+                execute_tools = row.get('should_execute_tool_calls', False)
+                if execute_tools:
+                    try:
+                        tool_calls = default_decode_execute_prompting(result)
+                    except Exception:
+                        tool_calls = None
+
+                    if tool_calls is None:
+                        break
+
+                    tool_outputs, _ = execute_multi_turn_func_call(
+                        tool_calls,
+                        initial_config=row['initial_config'],
+                        involved_classes=row['involved_classes'],
+                        model_name='evaluator_loop',
+                        test_entry_id=row['id'],
+                        long_context=('long_context' in row['test_category'] or 'composite' in row['test_category']),
+                        is_evaL_run=False,
+                    )
+                    # Append tool outputs to the current messages
+                    tool_results = []
+                    for tool_output, tool_call in zip(tool_outputs, tool_calls):
+                        tool_results.append({'role': 'tool', 'name': tool_call, 'content': tool_output})
+                    current_messages.append({
+                        'role': 'user',
+                        'content': repr(tool_results),
+                    })
+                else:
+                    break
+
+                n_steps += 1
+                if n_steps > MAXIMUM_STEP_LIMIT:
+                    logger.error(f'INFERENCE_ERROR: Exceeded max inference steps ({MAXIMUM_STEP_LIMIT})')
+                    break
+
+            all_model_responses.append(current_responses)
+
+        return all_model_responses
+
+    def generate_turn_with_tools(self, row: dict[str, Any], infer_cfg: dict[str, Any]) -> list[str]:
+        from bfcl_eval.constants.default_prompts import (DEFAULT_USER_PROMPT_FOR_ADDITIONAL_FUNCTION_FC,
+                                                         MAXIMUM_STEP_LIMIT)
+        from bfcl_eval.eval_checker.multi_turn_eval.multi_turn_utils import execute_multi_turn_func_call
+        from bfcl_eval.model_handler.utils import convert_to_function_call
+
+        all_model_responses = []
+        current_messages = []
+        turns = row['turns']
+        for turn_idx, messages in enumerate(turns):
+            n_steps = 0
+            current_responses = []
+            current_messages += messages.copy()
+            tools = row['tools']
+
+            if str(turn_idx) in row['missing_functions']:
+                assert len(messages) == 0, 'Holdout turn should not have user message.'
+                # inject new functions on the fly
+                new_tools = row['missing_functions'][str(turn_idx)]
+                for new_tool in new_tools:
+                    tools.append({
+                        'type': 'function',
+                        'function': new_tool[0],
+                    })
+                new_turn = [{
+                    'role': 'user',
+                    'content': DEFAULT_USER_PROMPT_FOR_ADDITIONAL_FUNCTION_FC,
+                }]
+                current_messages += new_turn
+
+            while True:
+                input_item = {
+                    'messages': current_messages,
+                    'tools': tools,
+                }
+                responses = self.process_single_input(input_item, infer_cfg)
+                message = responses['choices'][0]['message']
+
+                current_messages.append(message)
+                if isinstance(message, str):
+                    model_responses = [message]
+                    tool_call_strs = None
+                elif message.get('tool_calls'):
+                    model_responses = [{
+                        tc['function']['name']: tc['function']['arguments']
+                    } for tc in message['tool_calls']]
+                    try:
+                        tool_call_strs = convert_to_function_call(model_responses)
+                    except Exception as e:
+                        logger.error(f'Error converting tool calls to function call strings: {e}')
+                        tool_call_strs = None
+                else:
+                    model_responses = [message['content']]
+                    tool_call_strs = None
+
+                current_responses.extend(model_responses)
+
+                execute_tools = row.get('should_execute_tool_calls', False)
+                if execute_tools and tool_call_strs is not None:
+                    tool_outputs, _ = execute_multi_turn_func_call(
+                        tool_call_strs,
+                        initial_config=row['initial_config'],
+                        involved_classes=row['involved_classes'],
+                        model_name='evaluator_loop',
+                        test_entry_id=row['id'],
+                        long_context=('long_context' in row['test_category'] or 'composite' in row['test_category']),
+                        is_evaL_run=False,
+                    )
+
+                    for tc, tool_output in zip(message['tool_calls'], tool_outputs, strict=False):
+                        current_messages.append({
+                            'role': 'tool',
+                            'tool_call_id': tc['id'],
+                            'content': json.dumps({'response': tool_output}),
+                        })
+                else:
+                    break
+
+                n_steps += 1
+                if n_steps > MAXIMUM_STEP_LIMIT:
+                    logger.error(f'INFERENCE_ERROR: Exceeded max inference steps ({MAXIMUM_STEP_LIMIT})')
+                    break
+
+            all_model_responses.append(current_responses)
+
+        return all_model_responses

evalscope/models/adapters/server_adapter.py

@@ -1,11 +1,12 @@
+import copy
 import openai
 from collections import defaultdict
-from inspect import signature
 from openai.types.chat import ChatCompletion, ChatCompletionChunk
 from openai.types.chat.chat_completion import ChatCompletionMessage, Choice
 from typing import List, Optional, Union
 
 from evalscope.utils.logger import get_logger
+from evalscope.utils.utils import get_supported_params
 from .base_adapter import BaseModelAdapter
 
 logger = get_logger()
@@ -31,7 +32,7 @@ class ServerModelAdapter(BaseModelAdapter):
             api_key=api_key,
             base_url=self.api_url,
         )
-        self.supported_params = self._get_supported_params()
+        self.supported_params = get_supported_params(self.client.chat.completions.create)
 
         self.seed = kwargs.get('seed', None)
         self.timeout = kwargs.get('timeout', 60)
@@ -39,10 +40,6 @@ class ServerModelAdapter(BaseModelAdapter):
         self.model_cfg = {'api_url': api_url, 'model_id': model_id, 'api_key': api_key}
         super().__init__(model=None, model_cfg=self.model_cfg, **kwargs)
 
-    def _get_supported_params(self):
-        sig = signature(self.client.chat.completions.create)
-        return list(sig.parameters.keys())
-
     def predict(self, inputs: List[dict], infer_cfg: Optional[dict] = None) -> List[dict]:
         """
         Model prediction func.
@@ -65,18 +62,17 @@ class ServerModelAdapter(BaseModelAdapter):
 
     def process_single_input(self, input_item: dict, infer_cfg: dict) -> dict:
         """Process a single input item."""
-        if input_item.get('messages', None):
-            content = input_item['messages']
-        else:
-            content = self.make_request_content(input_item)
-        request_json = self.make_request(content, infer_cfg)
+        request_json = self.make_request(input_item, infer_cfg)
         response = self.send_request(request_json)
         return response
 
-    def make_request_content(self, input_item: dict) -> list:
+    def make_request_messages(self, input_item: dict) -> list:
         """
-        Make request content for OpenAI API.
+        Make request messages for OpenAI API.
         """
+        if input_item.get('messages', None):
+            return input_item['messages']
+
         data: list = input_item['data']
         if isinstance(data[0], tuple):  # for truthful_qa and hellaswag
             query = '\n'.join(''.join(item) for item in data)
@@ -93,18 +89,28 @@ class ServerModelAdapter(BaseModelAdapter):
 
         return messages
 
-    def make_request(self, content: list, infer_cfg: dict) -> dict:
+    def make_request(self, input_item: dict, infer_cfg: dict) -> dict:
         """Make request to remote API."""
+        messages = self.make_request_messages(input_item)
         # Format request JSON according to OpenAI API format
-        request_json = {'model': self.model_id, 'messages': content, **infer_cfg}
+        request_json = {'model': self.model_id, 'messages': messages, **infer_cfg}
 
         if self.timeout:
             request_json['timeout'] = self.timeout
 
+        request_json['stream'] = self.stream
         if self.stream:
-            request_json['stream'] = self.stream
             request_json['stream_options'] = {'include_usage': True}
 
+        if input_item.get('tools', None):
+            tools_copy = copy.deepcopy(input_item.get('tools'))
+            # Remove the "responses" from the functions, as that doesn't
+            # need to go to the model
+            for tool in tools_copy:
+                if 'function' in tool and 'response' in tool['function']:
+                    del tool['function']['response']
+            request_json['tools'] = tools_copy
+
         logger.debug(f'Request to remote API: {request_json}')
 
         return request_json
@@ -139,19 +145,65 @@ class ServerModelAdapter(BaseModelAdapter):
         collected_chunks = []
         collected_messages = defaultdict(list)
         collected_reasoning = defaultdict(list)
+        collected_tool_calls = defaultdict(dict)
 
         for chunk in response_stream:
             collected_chunks.append(chunk)
             for choice in chunk.choices:
+                # Handle reasoning content
                 if hasattr(choice.delta, 'reasoning_content') and choice.delta.reasoning_content is not None:
                     collected_reasoning[choice.index].append(choice.delta.reasoning_content)
+
+                # Handle regular content
                 if choice.delta.content is not None:
                     collected_messages[choice.index].append(choice.delta.content)
 
+                # Handle tool calls
+                if hasattr(choice.delta, 'tool_calls') and choice.delta.tool_calls:
+                    for tool_call in choice.delta.tool_calls:
+                        tool_id = tool_call.index
+
+                        # Initialize tool call if not present
+                        if tool_id not in collected_tool_calls[choice.index]:
+                            collected_tool_calls[choice.index][tool_id] = {
+                                'id': tool_call.id if hasattr(tool_call, 'id') and tool_call.id else None,
+                                'type': tool_call.type if hasattr(tool_call, 'type') and tool_call.type else None,
+                                'function': {
+                                    'name': '',
+                                    'arguments': ''
+                                }
+                            }
+
+                        # Update tool call with new chunks
+                        if hasattr(tool_call, 'function'):
+                            if hasattr(tool_call.function, 'name') and tool_call.function.name:
+                                collected_tool_calls[
+                                    choice.index][tool_id]['function']['name'] = tool_call.function.name
+
+                            if hasattr(tool_call.function, 'arguments') and tool_call.function.arguments:
+                                collected_tool_calls[
+                                    choice.index][tool_id]['function']['arguments'] += tool_call.function.arguments
+
+                        # Update ID if it was received later
+                        if hasattr(tool_call, 'id') and tool_call.id:
+                            collected_tool_calls[choice.index][tool_id]['id'] = tool_call.id
+
+        # Get all unique choice indices from all collections
+        all_indices = set(collected_messages.keys()) | set(collected_reasoning.keys()) | set(
+            collected_tool_calls.keys())
+
         choices = []
-        for index, messages in collected_messages.items():
-            full_reply_content = ''.join(messages)
-            reasoning = ''.join(collected_reasoning[index])
+        for index in all_indices:
+            full_reply_content = ''.join(collected_messages.get(index, []))
+            reasoning = ''.join(collected_reasoning.get(index, []))
+
+            # Process tool_calls for this choice if any exists
+            tool_calls_list = None
+            if index in collected_tool_calls and collected_tool_calls[index]:
+                tool_calls_list = list(collected_tool_calls[index].values())
+                # Filter out any tool calls with None id (incomplete tool calls)
+                tool_calls_list = [tc for tc in tool_calls_list if tc['id'] is not None]
+
             # use the finish_reason from the last chunk that generated this choice
             finish_reason = None
             for chunk in reversed(collected_chunks):
@@ -159,11 +211,16 @@ class ServerModelAdapter(BaseModelAdapter):
                     finish_reason = chunk.choices[0].finish_reason
                     break
 
+            message_kwargs = {'role': 'assistant', 'content': full_reply_content}
+
+            if reasoning:
+                message_kwargs['reasoning_content'] = reasoning
+
+            if tool_calls_list:
+                message_kwargs['tool_calls'] = tool_calls_list
+
             choice = Choice(
-                finish_reason=finish_reason or 'stop',
-                index=index,
-                message=ChatCompletionMessage(
-                    role='assistant', content=full_reply_content, reasoning_content=reasoning))
+                finish_reason=finish_reason or 'stop', index=index, message=ChatCompletionMessage(**message_kwargs))
             choices.append(choice)
 
         # build the final completion object

evalscope/models/custom/custom_model.py

@@ -10,9 +10,6 @@ class CustomModel(ABC):
         self.config = config
         self.kwargs = kwargs
 
-        if config.get('model_id', None) is None:
-            raise ValueError(f'**Error: model_id is required in config for CustomModel. Got config: {config}')
-
     @abstractmethod
     @torch.no_grad()
     def predict(self, prompts: List[str], **kwargs) -> List[Dict[str, Any]]:

evalscope/models/custom/dummy_model.py

@@ -1,61 +1,99 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
-import os
 import time
 from typing import List
 
+from evalscope.models import CustomModel
 from evalscope.utils.logger import get_logger
-from .custom_model import CustomModel
 
 logger = get_logger()
-"""
-This script is used to rewrite the evaluation results without re-running the model predictions.
-"""
 
 
 class DummyCustomModel(CustomModel):
 
-    def __init__(self, config: dict = {'model_id': 'dummy-model'}, **kwargs):
+    def __init__(self, config: dict = {}, **kwargs):
         super(DummyCustomModel, self).__init__(config=config, **kwargs)
 
-    def predict(self, prompts: List[dict], **kwargs):
-        # ONLY FOR DUMMY IMPLEMENTATION, DO NOT EDIT OR USE IN PRODUCTION.
+    def make_request_messages(self, input_item: dict) -> list:
+        """
+        Make request messages for OpenAI API.
+        """
+        if input_item.get('messages', None):
+            return input_item['messages']
 
-        response = ''
+        data: list = input_item['data']
+        if isinstance(data[0], tuple):  # for truthful_qa and hellaswag
+            query = '\n'.join(''.join(item) for item in data)
+            system_prompt = input_item.get('system_prompt', None)
+        else:
+            query = data[0]
+            system_prompt = input_item.get('system_prompt', None)
 
-        res_d: dict = {
-            'choices': [{
-                'index': 0,
-                'message': {
-                    'content': response,
-                    'role': 'assistant'
-                }
-            }],
-            'created': time.time(),
-            'model': self.config.get('model_id'),  # should be model_id
-            'object': 'chat.completion',
-            'usage': {
-                'completion_tokens': 0,
-                'prompt_tokens': 0,
-                'total_tokens': 0
-            }
-        }
+        messages = []
+        if system_prompt:
+            messages.append({'role': 'system', 'content': system_prompt})
 
-        return [res_d for _ in prompts]
+        messages.append({'role': 'user', 'content': query})
 
+        return messages
 
-if __name__ == '__main__':
-    from evalscope.run import run_task
-    from evalscope.utils.io_utils import yaml_to_dict
-
-    # step1: 如果outputs做了迁移，需要修改outputs/eval_xxx 中的configs/task_output_config.yaml中的路径配置
-    # step2: 执行此脚本，默认使用use_cache=True，实现免推理对eval结果进行刷新
+    def predict(self, prompts: List[dict], **kwargs):
+        original_inputs = kwargs.get('origin_inputs', None)
+        infer_cfg = kwargs.get('infer_cfg', None)
+
+        logger.debug(f'** Prompts: {prompts}')
+        if original_inputs is not None:
+            logger.debug(f'** Original inputs: {original_inputs}')
+        if infer_cfg is not None:
+            logger.debug(f'** Inference config: {infer_cfg}')
+
+        # Simulate a response based on the prompts
+        # Must return a list of dicts with the same format as the OpenAI API.
+        responses = []
+        for input_item in original_inputs:
+            message = self.make_request_messages(input_item)
+            response = f'Dummy response for prompt: {message}'
+
+            res_d = {
+                'choices': [{
+                    'index': 0,
+                    'message': {
+                        'content': response,
+                        'role': 'assistant'
+                    }
+                }],
+                'created': time.time(),
+                'model': self.config.get('model_id'),
+                'object': 'chat.completion',
+                'usage': {
+                    'completion_tokens': 0,
+                    'prompt_tokens': 0,
+                    'total_tokens': 0
+                }
+            }
 
-    swift_model = DummyCustomModel(config={'model_id': 'swift-model-dummy'})
+            responses.append(res_d)
 
-    task_cfg_file = '/path/to/eval_your_model_results/configs/task_output_config.yaml'
+        return responses
 
-    task_cfg_d = yaml_to_dict(task_cfg_file)
-    task_cfg_d.update({'model': swift_model})
 
-    eval_results: dict = run_task(task_cfg=task_cfg_d)
-    print('** Evaluation results finished !\n')
+if __name__ == '__main__':
+    from evalscope import TaskConfig, run_task
+
+    dummy_model = DummyCustomModel()
+    task_config = TaskConfig(
+        model=dummy_model,
+        model_id='evalscope-model-dummy',
+        datasets=['gsm8k'],
+        eval_type='custom',  # must be custom for custom model evaluation
+        generation_config={
+            'max_new_tokens': 100,
+            'temperature': 0.0,
+            'top_p': 1.0,
+            'top_k': 50,
+            'repetition_penalty': 1.0
+        },
+        debug=True,
+        limit=5,
+    )
+
+    eval_results = run_task(task_cfg=task_config)

evalscope/models/local_model.py

@@ -82,7 +82,7 @@ class LocalImageModel(LocalModel):
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
 
-        self.pipeline_cls = kwargs.pop('pipeline_cls', None)
+        self.pipeline_cls = self.kwargs.pop('pipeline_cls', None)
         # default to DiffusionPipeline if not specified
         if self.pipeline_cls is None:
             if 'flux' in self.model_id.lower():

evalscope/models/register.py

@@ -47,8 +47,9 @@ def register_model_adapter_class(cls, name=None):
 # register all model adapters
 register_model_adapter_class(BaseModelAdapter, name='base')
 register_model_adapter_class(ChatGenerationModelAdapter, name=OutputType.GENERATION)
-register_model_adapter_class(ContinuationLogitsModelAdapter, name=OutputType.LOGITS)
+register_model_adapter_class(ContinuationLogitsModelAdapter, name=OutputType.CONTINUOUS)
 register_model_adapter_class(MultiChoiceModelAdapter, name=OutputType.MULTIPLE_CHOICE)
 register_model_adapter_class(CustomModelAdapter, name='custom')
 register_model_adapter_class(ServerModelAdapter, name='server')
+register_model_adapter_class(BFCLAdapter, name='bfcl_server')
 register_model_adapter_class(T2IModelAdapter, name=OutputType.IMAGE_GENERATION)

evalscope/perf/arguments.py

@@ -55,13 +55,14 @@ class Arguments:
 
     # Response settings
     frequency_penalty: Optional[float] = None  # Frequency penalty for the response
+    repetition_penalty: Optional[float] = None  # Repetition penalty for the response
     logprobs: Optional[bool] = None  # Whether to log probabilities
     max_tokens: Optional[int] = 2048  # Maximum number of tokens in the response
     min_tokens: Optional[int] = None  # Minimum number of tokens in the response
     n_choices: Optional[int] = None  # Number of response choices
     seed: Optional[int] = 0  # Random seed for reproducibility
-    stop: Optional[List[str]] = field(default_factory=list)  # Stop sequences for the response
-    stop_token_ids: Optional[List[str]] = field(default_factory=list)  # Stop token IDs for the response
+    stop: Optional[List[str]] = None  # Stop sequences for the response
+    stop_token_ids: Optional[List[str]] = None  # Stop token IDs for the response
     stream: Optional[bool] = True  # Whether to stream the response
     temperature: float = 0.0  # Temperature setting for the response
     top_p: Optional[float] = None  # Top-p (nucleus) sampling setting for the response
@@ -181,6 +182,7 @@ def add_argument(parser: argparse.ArgumentParser):
 
     # Response settings
     parser.add_argument('--frequency-penalty', type=float, help='The frequency_penalty value', default=None)
+    parser.add_argument('--repetition-penalty', type=float, help='The repetition_penalty value', default=None)
     parser.add_argument('--logprobs', action='store_true', help='The logprobs', default=None)
     parser.add_argument(
         '--max-tokens', type=int, help='The maximum number of tokens that can be generated', default=2048)