evalscope 0.16.1__py3-none-any.whl → 0.16.2__py3-none-any.whl

evalscope/evaluator/evaluator.py

@@ -13,7 +13,7 @@ from evalscope.benchmarks import DataAdapter
 from evalscope.config import TaskConfig
 from evalscope.constants import AnswerKeys, DumpMode, EvalStage, EvalType, JudgeStrategy, ReviewKeys
 from evalscope.models import BaseModelAdapter
-from evalscope.report import Report, gen_report_table
+from evalscope.report import Report, gen_table
 from evalscope.utils import dict_torch_dtype_to_str, gen_hash
 from evalscope.utils.io_utils import OutputsStructure, dump_jsonl_data, jsonl_to_list
 from evalscope.utils.logger import get_logger
@@ -108,7 +108,6 @@ class Evaluator(object):
         return answer_d
 
     def _get_answer(self, input_prompts, subset_name, infer_cfg) -> List[dict]:
-        answers_list = []
         try:
             # get answer from model
             answer_ds: List[dict] = self.model_adapter.predict(inputs=input_prompts, infer_cfg=infer_cfg)
@@ -117,10 +116,11 @@ class Evaluator(object):
             # if ignore_errors is True, continue to next input
             if self.task_cfg.ignore_errors:
                 logger.warning('`ignore_errors` is set to True. Dropping this prompt and continuing with evaluation.')
-                return answers_list
+                return []
             else:
                 raise e
         # process answer
+        answers_list = []
         for answer_d, input_prompt in zip(answer_ds, input_prompts):
             answer_id = self._generate_answer_id(self.model_adapter.model_cfg, input_prompt, infer_cfg)
             processed_answer = self._process_answer(answer_d, input_prompt, subset_name, answer_id)
@@ -399,8 +399,9 @@ class Evaluator(object):
 
         # Make table
         try:
-            report_table = gen_report_table(report_map)
-            logger.info(f'{self.dataset_name_or_path} report table: \n{report_table} \n')
+            report_table = gen_table(report_list=[report_map], add_overall_metric=True)
+            logger.info(f'\n{self.dataset_name_or_path} report table:'
+                        f'\n{report_table} \n')
         except Exception:
             logger.error('Failed to generate report table.')
 

evalscope/metrics/t2v_metrics/__init__.py

@@ -1,66 +1,52 @@
-from __future__ import absolute_import, division, print_function
-
-from .clipscore import CLIPScore, list_all_clipscore_models
-from .constants import CACHE_DIR
-from .itmscore import ITMScore, list_all_itmscore_models
-from .vqascore import VQAScore, list_all_vqascore_models
-
-
-def list_all_models():
-    return list_all_vqascore_models() + list_all_clipscore_models() + list_all_itmscore_models()
-
-
-def get_score_model(model='clip-flant5-xxl', device='cuda', cache_dir=CACHE_DIR, **kwargs):
-    if model in list_all_vqascore_models():
-        return VQAScore(model, device=device, cache_dir=cache_dir, **kwargs)
-    elif model in list_all_clipscore_models():
-        return CLIPScore(model, device=device, cache_dir=cache_dir, **kwargs)
-    elif model in list_all_itmscore_models():
-        return ITMScore(model, device=device, cache_dir=cache_dir, **kwargs)
-    else:
-        raise NotImplementedError()
-
-
 def clip_flant5_score():
+    from .vqascore import VQAScore
     clip_flant5_score = VQAScore(model='clip-flant5-xxl')
     return clip_flant5_score
 
 
 def pick_score():
+    from .clipscore import CLIPScore
     pick_score = CLIPScore(model='pickscore-v1')
     return pick_score
 
 
 def clip_score():
+    from .clipscore import CLIPScore
     clip_score = CLIPScore(model='openai:ViT-L-14-336')
     return clip_score
 
 
 def blip2_score():
+    from .itmscore import ITMScore
     blip_itm_score = ITMScore(model='blip2-itm')
     return blip_itm_score
 
 
 def hpsv2_score():
+    from .clipscore import CLIPScore
     hpsv2_score = CLIPScore(model='hpsv2')
     return hpsv2_score
 
 
 def hpsv2_1_score():
+    from .clipscore import CLIPScore
     hpsv2_1_score = CLIPScore(model='hpsv2.1')
     return hpsv2_1_score
 
 
 def image_reward_score():
+    from .itmscore import ITMScore
     image_reward_score = ITMScore(model='image-reward-v1')
     return image_reward_score
 
 
 def fga_blip2_score():
+    from .itmscore import ITMScore
     fga_blip2_score = ITMScore(model='fga_blip2')
     return fga_blip2_score
 
 
 def mps_score():
+    from .clipscore import CLIPScore
     mps_score = CLIPScore(model='mps')
     return mps_score

evalscope/models/adapters/__init__.py

@@ -1,4 +1,5 @@
 from .base_adapter import BaseModelAdapter, initialize_model_adapter
+from .bfcl_adapter import BFCLAdapter
 from .chat_adapter import ChatGenerationModelAdapter
 from .choice_adapter import ContinuationLogitsModelAdapter, MultiChoiceModelAdapter
 from .custom_adapter import CustomModelAdapter
@@ -13,5 +14,6 @@ __all__ = [
     'MultiChoiceModelAdapter',
     'CustomModelAdapter',
     'ServerModelAdapter',
+    'BFCLAdapter',
     'T2IModelAdapter',
 ]

evalscope/models/adapters/base_adapter.py

@@ -44,35 +44,39 @@ def initialize_model_adapter(task_cfg: 'TaskConfig', benchmark: 'DataAdapter', b
             raise ValueError(f'Expected evalscope.models.custom.CustomModel, but got {type(task_cfg.model)}.')
         from evalscope.models import CustomModelAdapter
         return CustomModelAdapter(custom_model=task_cfg.model)
-    elif task_cfg.eval_type == EvalType.SERVICE or task_cfg.api_url is not None:
-        from evalscope.models import ServerModelAdapter
-
-        if benchmark.model_adapter in [OutputType.CONTINUOUS, OutputType.MULTIPLE_CHOICE]:
-            logger.warning('Output type is set to logits. This is not supported for service evaluation. '
-                           'Setting output type to generation by default.')
-            benchmark.model_adapter = OutputType.GENERATION
-
-        return ServerModelAdapter(
-            api_url=task_cfg.api_url,
-            model_id=task_cfg.model,
-            api_key=task_cfg.api_key,
-            seed=task_cfg.seed,
-            timeout=task_cfg.timeout,
-            stream=task_cfg.stream,
-        )
     else:
         from ..register import get_model_adapter
 
-        # for local model, we need to determine the model adapter class based on the output type
+        # we need to determine the model adapter class based on the output type
         model_adapter_cls_str = benchmark.model_adapter
-        if model_adapter_cls_str not in benchmark.output_types:
-            logger.warning(f'Output type {model_adapter_cls_str} is not supported for benchmark {benchmark.name}. '
-                           f'Using {benchmark.output_types[0]} instead.')
-            model_adapter_cls_str = benchmark.output_types[0]
 
-        model_adapter_cls = get_model_adapter(model_adapter_cls_str)
-        return model_adapter_cls(
-            model=base_model,
-            generation_config=task_cfg.generation_config,
-            chat_template=task_cfg.chat_template,
-            task_cfg=task_cfg)
+        if task_cfg.eval_type == EvalType.SERVICE or task_cfg.api_url is not None:
+
+            if 'server' not in model_adapter_cls_str:
+                model_adapter_cls_str = 'server'
+                logger.info(
+                    f'Using {model_adapter_cls.__name__} for api model evaluation for benchmark {benchmark.name}.')
+
+            # init server model adapter
+            model_adapter_cls = get_model_adapter(model_adapter_cls_str)
+
+            return model_adapter_cls(
+                api_url=task_cfg.api_url,
+                model_id=task_cfg.model,
+                api_key=task_cfg.api_key,
+                seed=task_cfg.seed,
+                timeout=task_cfg.timeout,
+                stream=task_cfg.stream,
+            )
+        else:
+            if model_adapter_cls_str not in benchmark.output_types:
+                logger.warning(f'Output type {model_adapter_cls_str} is not supported for benchmark {benchmark.name}.'
+                               f'Using {benchmark.output_types[0]} instead.')
+                model_adapter_cls_str = benchmark.output_types[0]
+
+            model_adapter_cls = get_model_adapter(model_adapter_cls_str)
+            return model_adapter_cls(
+                model=base_model,
+                generation_config=task_cfg.generation_config,
+                chat_template=task_cfg.chat_template,
+                task_cfg=task_cfg)

evalscope/models/adapters/bfcl_adapter.py

@@ -0,0 +1,244 @@
+import json
+import time
+import uuid
+from typing import Any, List, Optional, Union
+
+from evalscope.utils.logger import get_logger
+from .server_adapter import ServerModelAdapter
+
+logger = get_logger()
+
+
+class BFCLAdapter(ServerModelAdapter):
+    """
+    BFCL model adapter to request remote API model and generate results for BFCL evaluation.
+    Support multi-turn and single-turn function calling tasks.
+    """
+
+    def __init__(self, api_url: str, model_id: str, api_key: str = 'EMPTY', **kwargs):
+        """
+        Args:
+            api_url: The URL of the remote API model.
+            model_id: The ID of the remote API model.
+            api_key: The API key of the remote API model.
+        """
+        super().__init__(api_url=api_url, model_id=model_id, api_key=api_key, **kwargs)
+
+    def predict(self, inputs: List[dict], infer_cfg: Optional[dict] = None) -> List[dict]:
+        """
+        Model prediction func. For multi-turn evals, we pass a list[list[message]] to the model
+        where each list is a follow up turn in the conversation
+        each turn is a List[List[Message]]
+
+        Args:
+            inputs (List[dict]): The input data.
+            infer_cfg (dict): Inference configuration.
+
+        Returns:
+            res (List[dict]): The model prediction results.
+        """
+        infer_cfg = infer_cfg or {}
+        results = []
+
+        for input_item in inputs:
+            # This flag decides if we pass tools to the API or try tool calling via prompting
+            # Passing tools to the API means that we rely on the API to manage system prompt specifics
+            # and also expect parsed tool calls in the ChatCompletionMessage object
+            # This is how the is_fc_model=True benchmark is designed to work
+            # On the other hand, we try to manage
+            # tool calling via prompting and parse tool calls in the standard text response
+            # This is how the is_fc_model=False benchmark is designed to work
+            row = input_item.get('messages')
+            is_fc_model = row.get('is_fc_model', False)
+
+            if is_fc_model:
+                response = self.generate_turn_with_tools(row, infer_cfg)
+            else:
+                response = self.generate_turn(row, infer_cfg)
+
+            # wrap response with openai types
+            res_d = {
+                'choices': [{
+                    'index': 0,
+                    'message': {
+                        'content': response,
+                        'role': 'assistant'
+                    }
+                }],
+                'created': time.time(),
+                'model': self.model_id,
+                'object': 'chat.completion',
+                'usage': {
+                    'completion_tokens': 0,
+                    'prompt_tokens': 0,
+                    'total_tokens': 0
+                }
+            }
+            results.append(res_d)
+
+        return results
+
+    def generate_turn(self, row: dict[str, Any], infer_cfg: dict[str, Any]) -> list[str]:
+        from bfcl_eval.constants.default_prompts import (DEFAULT_USER_PROMPT_FOR_ADDITIONAL_FUNCTION_PROMPTING,
+                                                         MAXIMUM_STEP_LIMIT)
+        from bfcl_eval.eval_checker.multi_turn_eval.multi_turn_utils import execute_multi_turn_func_call
+        from bfcl_eval.model_handler.utils import default_decode_execute_prompting
+
+        all_model_responses = []
+        current_messages = []
+        turns = row['turns']
+        for turn_idx, messages in enumerate(turns):
+            n_steps = 0
+            current_responses = []
+            current_messages += messages.copy()
+
+            if str(turn_idx) in row['missing_functions']:
+                assert len(messages) == 0, 'Holdout turn should not have user message.'
+                new_turn = [{
+                    'role':
+                    'user',
+                    'content':
+                    DEFAULT_USER_PROMPT_FOR_ADDITIONAL_FUNCTION_PROMPTING.format(
+                        functions=row['missing_functions'][str(turn_idx)]),
+                }]
+                current_messages += new_turn
+
+            while True:
+                input_item = {
+                    'messages': current_messages,
+                }
+                responses = self.process_single_input(input_item, infer_cfg)
+                result = responses['choices'][0]['message']['content']
+
+                logger.debug(f'Turn:{turn_idx} Step:{n_steps} Result: {result}')
+                current_messages.append({
+                    'role': 'assistant',
+                    'content': result,
+                })
+                current_responses.append(result)
+
+                execute_tools = row.get('should_execute_tool_calls', False)
+                if execute_tools:
+                    try:
+                        tool_calls = default_decode_execute_prompting(result)
+                    except Exception:
+                        tool_calls = None
+
+                    if tool_calls is None:
+                        break
+
+                    tool_outputs, _ = execute_multi_turn_func_call(
+                        tool_calls,
+                        initial_config=row['initial_config'],
+                        involved_classes=row['involved_classes'],
+                        model_name='evaluator_loop',
+                        test_entry_id=row['id'],
+                        long_context=('long_context' in row['test_category'] or 'composite' in row['test_category']),
+                        is_evaL_run=False,
+                    )
+                    # Append tool outputs to the current messages
+                    tool_results = []
+                    for tool_output, tool_call in zip(tool_outputs, tool_calls):
+                        tool_results.append({'role': 'tool', 'name': tool_call, 'content': tool_output})
+                    current_messages.append({
+                        'role': 'user',
+                        'content': repr(tool_results),
+                    })
+                else:
+                    break
+
+                n_steps += 1
+                if n_steps > MAXIMUM_STEP_LIMIT:
+                    logger.error(f'INFERENCE_ERROR: Exceeded max inference steps ({MAXIMUM_STEP_LIMIT})')
+                    break
+
+            all_model_responses.append(current_responses)
+
+        return all_model_responses
+
+    def generate_turn_with_tools(self, row: dict[str, Any], infer_cfg: dict[str, Any]) -> list[str]:
+        from bfcl_eval.constants.default_prompts import (DEFAULT_USER_PROMPT_FOR_ADDITIONAL_FUNCTION_FC,
+                                                         MAXIMUM_STEP_LIMIT)
+        from bfcl_eval.eval_checker.multi_turn_eval.multi_turn_utils import execute_multi_turn_func_call
+        from bfcl_eval.model_handler.utils import convert_to_function_call
+
+        all_model_responses = []
+        current_messages = []
+        turns = row['turns']
+        for turn_idx, messages in enumerate(turns):
+            n_steps = 0
+            current_responses = []
+            current_messages += messages.copy()
+            tools = row['tools']
+
+            if str(turn_idx) in row['missing_functions']:
+                assert len(messages) == 0, 'Holdout turn should not have user message.'
+                # inject new functions on the fly
+                new_tools = row['missing_functions'][str(turn_idx)]
+                for new_tool in new_tools:
+                    tools.append({
+                        'type': 'function',
+                        'function': new_tool[0],
+                    })
+                new_turn = [{
+                    'role': 'user',
+                    'content': DEFAULT_USER_PROMPT_FOR_ADDITIONAL_FUNCTION_FC,
+                }]
+                current_messages += new_turn
+
+            while True:
+                input_item = {
+                    'messages': current_messages,
+                    'tools': tools,
+                }
+                responses = self.process_single_input(input_item, infer_cfg)
+                message = responses['choices'][0]['message']
+
+                current_messages.append(message)
+                if isinstance(message, str):
+                    model_responses = [message]
+                    tool_call_strs = None
+                elif message.get('tool_calls'):
+                    model_responses = [{
+                        tc['function']['name']: tc['function']['arguments']
+                    } for tc in message['tool_calls']]
+                    try:
+                        tool_call_strs = convert_to_function_call(model_responses)
+                    except Exception as e:
+                        logger.error(f'Error converting tool calls to function call strings: {e}')
+                        tool_call_strs = None
+                else:
+                    model_responses = [message['content']]
+                    tool_call_strs = None
+
+                current_responses.extend(model_responses)
+
+                execute_tools = row.get('should_execute_tool_calls', False)
+                if execute_tools and tool_call_strs is not None:
+                    tool_outputs, _ = execute_multi_turn_func_call(
+                        tool_call_strs,
+                        initial_config=row['initial_config'],
+                        involved_classes=row['involved_classes'],
+                        model_name='evaluator_loop',
+                        test_entry_id=row['id'],
+                        long_context=('long_context' in row['test_category'] or 'composite' in row['test_category']),
+                        is_evaL_run=False,
+                    )
+
+                    for tc, tool_output in zip(message['tool_calls'], tool_outputs, strict=False):
+                        current_messages.append({
+                            'role': 'tool',
+                            'tool_call_id': tc['id'],
+                            'content': json.dumps({'response': tool_output}),
+                        })
+                else:
+                    break
+
+                n_steps += 1
+                if n_steps > MAXIMUM_STEP_LIMIT:
+                    logger.error(f'INFERENCE_ERROR: Exceeded max inference steps ({MAXIMUM_STEP_LIMIT})')
+                    break
+
+            all_model_responses.append(current_responses)
+
+        return all_model_responses

evalscope/models/adapters/server_adapter.py

@@ -1,3 +1,4 @@
+import copy
 import openai
 from collections import defaultdict
 from openai.types.chat import ChatCompletion, ChatCompletionChunk
@@ -61,18 +62,17 @@ class ServerModelAdapter(BaseModelAdapter):
 
     def process_single_input(self, input_item: dict, infer_cfg: dict) -> dict:
         """Process a single input item."""
-        if input_item.get('messages', None):
-            content = input_item['messages']
-        else:
-            content = self.make_request_content(input_item)
-        request_json = self.make_request(content, infer_cfg)
+        request_json = self.make_request(input_item, infer_cfg)
         response = self.send_request(request_json)
         return response
 
-    def make_request_content(self, input_item: dict) -> list:
+    def make_request_messages(self, input_item: dict) -> list:
         """
-        Make request content for OpenAI API.
+        Make request messages for OpenAI API.
         """
+        if input_item.get('messages', None):
+            return input_item['messages']
+
         data: list = input_item['data']
         if isinstance(data[0], tuple):  # for truthful_qa and hellaswag
             query = '\n'.join(''.join(item) for item in data)
@@ -89,18 +89,28 @@ class ServerModelAdapter(BaseModelAdapter):
 
         return messages
 
-    def make_request(self, content: list, infer_cfg: dict) -> dict:
+    def make_request(self, input_item: dict, infer_cfg: dict) -> dict:
         """Make request to remote API."""
+        messages = self.make_request_messages(input_item)
         # Format request JSON according to OpenAI API format
-        request_json = {'model': self.model_id, 'messages': content, **infer_cfg}
+        request_json = {'model': self.model_id, 'messages': messages, **infer_cfg}
 
         if self.timeout:
             request_json['timeout'] = self.timeout
 
+        request_json['stream'] = self.stream
         if self.stream:
-            request_json['stream'] = self.stream
             request_json['stream_options'] = {'include_usage': True}
 
+        if input_item.get('tools', None):
+            tools_copy = copy.deepcopy(input_item.get('tools'))
+            # Remove the "responses" from the functions, as that doesn't
+            # need to go to the model
+            for tool in tools_copy:
+                if 'function' in tool and 'response' in tool['function']:
+                    del tool['function']['response']
+            request_json['tools'] = tools_copy
+
         logger.debug(f'Request to remote API: {request_json}')
 
         return request_json
@@ -135,19 +145,65 @@ class ServerModelAdapter(BaseModelAdapter):
         collected_chunks = []
         collected_messages = defaultdict(list)
         collected_reasoning = defaultdict(list)
+        collected_tool_calls = defaultdict(dict)
 
         for chunk in response_stream:
             collected_chunks.append(chunk)
             for choice in chunk.choices:
+                # Handle reasoning content
                 if hasattr(choice.delta, 'reasoning_content') and choice.delta.reasoning_content is not None:
                     collected_reasoning[choice.index].append(choice.delta.reasoning_content)
+
+                # Handle regular content
                 if choice.delta.content is not None:
                     collected_messages[choice.index].append(choice.delta.content)
 
+                # Handle tool calls
+                if hasattr(choice.delta, 'tool_calls') and choice.delta.tool_calls:
+                    for tool_call in choice.delta.tool_calls:
+                        tool_id = tool_call.index
+
+                        # Initialize tool call if not present
+                        if tool_id not in collected_tool_calls[choice.index]:
+                            collected_tool_calls[choice.index][tool_id] = {
+                                'id': tool_call.id if hasattr(tool_call, 'id') and tool_call.id else None,
+                                'type': tool_call.type if hasattr(tool_call, 'type') and tool_call.type else None,
+                                'function': {
+                                    'name': '',
+                                    'arguments': ''
+                                }
+                            }
+
+                        # Update tool call with new chunks
+                        if hasattr(tool_call, 'function'):
+                            if hasattr(tool_call.function, 'name') and tool_call.function.name:
+                                collected_tool_calls[
+                                    choice.index][tool_id]['function']['name'] = tool_call.function.name
+
+                            if hasattr(tool_call.function, 'arguments') and tool_call.function.arguments:
+                                collected_tool_calls[
+                                    choice.index][tool_id]['function']['arguments'] += tool_call.function.arguments
+
+                        # Update ID if it was received later
+                        if hasattr(tool_call, 'id') and tool_call.id:
+                            collected_tool_calls[choice.index][tool_id]['id'] = tool_call.id
+
+        # Get all unique choice indices from all collections
+        all_indices = set(collected_messages.keys()) | set(collected_reasoning.keys()) | set(
+            collected_tool_calls.keys())
+
         choices = []
-        for index, messages in collected_messages.items():
-            full_reply_content = ''.join(messages)
-            reasoning = ''.join(collected_reasoning[index])
+        for index in all_indices:
+            full_reply_content = ''.join(collected_messages.get(index, []))
+            reasoning = ''.join(collected_reasoning.get(index, []))
+
+            # Process tool_calls for this choice if any exists
+            tool_calls_list = None
+            if index in collected_tool_calls and collected_tool_calls[index]:
+                tool_calls_list = list(collected_tool_calls[index].values())
+                # Filter out any tool calls with None id (incomplete tool calls)
+                tool_calls_list = [tc for tc in tool_calls_list if tc['id'] is not None]
+
             # use the finish_reason from the last chunk that generated this choice
             finish_reason = None
             for chunk in reversed(collected_chunks):
@@ -155,11 +211,16 @@ class ServerModelAdapter(BaseModelAdapter):
                     finish_reason = chunk.choices[0].finish_reason
                     break
 
+            message_kwargs = {'role': 'assistant', 'content': full_reply_content}
+
+            if reasoning:
+                message_kwargs['reasoning_content'] = reasoning
+
+            if tool_calls_list:
+                message_kwargs['tool_calls'] = tool_calls_list
+
             choice = Choice(
-                finish_reason=finish_reason or 'stop',
-                index=index,
-                message=ChatCompletionMessage(
-                    role='assistant', content=full_reply_content, reasoning_content=reasoning))
+                finish_reason=finish_reason or 'stop', index=index, message=ChatCompletionMessage(**message_kwargs))
             choices.append(choice)
 
         # build the final completion object

evalscope/models/custom/custom_model.py

@@ -10,9 +10,6 @@ class CustomModel(ABC):
         self.config = config
         self.kwargs = kwargs
 
-        if config.get('model_id', None) is None:
-            raise ValueError(f'**Error: model_id is required in config for CustomModel. Got config: {config}')
-
     @abstractmethod
     @torch.no_grad()
     def predict(self, prompts: List[str], **kwargs) -> List[Dict[str, Any]]: