evalscope 0.17.1__py3-none-any.whl → 1.0.0__py3-none-any.whl

evalscope/metrics/completion_parsers.py

@@ -1,227 +0,0 @@
-# Copyright (c) Alibaba, Inc. and its affiliates.
-# flake8: noqa
-
-import ast
-import re
-
-# from . import utils as ann_utils
-from evalscope.constants import ArenaWinner
-from evalscope.utils.logger import get_logger
-
-logger = get_logger()
-
-one_score_pattern = re.compile('\[\[(\d+\.?\d*)\]\]')
-one_score_pattern_backup = re.compile('\[(\d+\.?\d*)\]')
-
-
-# modified from: https://github.com/lm-sys/FastChat/blob/main/fastchat/eval/eval_gpt_review.py#L47
-# does not work with batched completions
-def lmsys_parser(completion, output_format):
-    if output_format == '[[rating]]':
-        match = re.search(one_score_pattern, completion)
-        if not match:
-            match = re.search(one_score_pattern_backup, completion)
-
-        if match:
-            rating = ast.literal_eval(match.groups()[0])
-        else:
-            logger.error(f'Content: {completion}\n'
-                         'You must manually fix the score.')
-            rating = -1
-
-        return rating
-    if output_format == '[[rating_a,rating_b]]':
-        try:
-            score_pair = completion.split('\n')[0]
-            score_pair = score_pair.replace(',', ' ')
-            sp = score_pair.split(' ')
-            if len(sp) == 2:
-                score_1 = float(sp[0])
-                score_2 = float(sp[1])
-                if score_1 > score_2:
-                    winner = ArenaWinner.MODEL_A
-                elif score_1 < score_2:
-                    winner = ArenaWinner.MODEL_B
-                else:
-                    if score_1 == score_1 == -1:
-                        winner = ArenaWinner.UNKNOWN
-                    winner = ArenaWinner.TIE
-                return winner, [score_1, score_2]
-            else:
-                raise Exception('Invalid score pair.')
-        except Exception as e:
-            logger.error(f'{e}\nContent: {completion}\nYou must manually fix the score pair.')
-            return ArenaWinner.UNKNOWN, [-1, -1]
-    elif output_format == '[[A]]':
-        if '[[A]]' in completion:
-            winner = ArenaWinner.MODEL_A
-        elif '[[B]]' in completion:
-            winner = ArenaWinner.MODEL_B
-        elif '[[C]]' in completion:
-            winner = ArenaWinner.TIE
-        else:
-            logger.error(f'\nContent: {completion}\nYou must manually fix the score.')
-            winner = ArenaWinner.UNKNOWN
-        return winner
-
-
-def ranking_parser(completion, **kwargs):
-    try:
-        if isinstance(completion, str):
-            ordered_completions = ast.literal_eval(completion)
-        else:
-            ordered_completions = completion
-
-        rank = [c for c in ordered_completions if c['model'] == 'model_a'][0]['rank']
-        assert rank in [1, 2]
-
-        return ArenaWinner.MODEL_A if rank == 1 else ArenaWinner.MODEL_B
-    except Exception as e:
-        logger.error(f'{e}\nContent: {completion}\n'
-                     'You must manually fix the score pair.')
-        return ArenaWinner.UNKNOWN
-
-
-class ResponseParser:
-
-    @staticmethod
-    def parse_first_capital(text: str, options: list[str]) -> str:
-        for t in text:
-            if t.isupper() and (t in options):
-                return t
-        return ''
-
-    @staticmethod
-    def parse_last_capital(text: str, options: list[str]) -> str:
-        for t in text[::-1]:
-            if t.isupper() and (t in options):
-                return t
-        return ''
-
-    @staticmethod
-    def parse_first_option_with_choices(text: str, options: list[str]) -> str:
-        """
-        Find first valid option for text.
-
-        Args:
-            text: The text to parse.
-            options: The options to find. e.g. ['A', 'B', 'C', 'D']
-        """
-        options_concat = ResponseParser.process_options(options)
-
-        patterns = [
-            rf'答案是?\s?([{options_concat}])',
-            rf'答案是?\s?：([{options_concat}])',
-            rf'答案是?\s?:([{options_concat}])',
-            rf'答案应该?是\s?([{options_concat}])',
-            rf'答案应该?选\s?([{options_concat}])',
-            rf'答案为\s?([{options_concat}])',
-            rf'答案选\s?([{options_concat}])',
-            rf'选择?\s?([{options_concat}])',
-            rf'故选?\s?([{options_concat}])'
-            rf'只有选?项?\s?([{options_concat}])\s?是?对',
-            rf'只有选?项?\s?([{options_concat}])\s?是?错',
-            rf'只有选?项?\s?([{options_concat}])\s?不?正确',
-            rf'只有选?项?\s?([{options_concat}])\s?错误',
-            rf'说法不?对选?项?的?是\s?([{options_concat}])',
-            rf'说法不?正确选?项?的?是\s?([{options_concat}])',
-            rf'说法错误选?项?的?是\s?([{options_concat}])',
-            rf'([{options_concat}])\s?是正确的',
-            rf'([{options_concat}])\s?是正确答案',
-            rf'选项\s?([{options_concat}])\s?正确',
-            rf'所以答\s?([{options_concat}])',
-            rf'所以\s?([{options_concat}][.。$]?$)',
-            rf'所有\s?([{options_concat}][.。$]?$)',
-            rf'[\s，：:,]([{options_concat}])[。，,\.]?$',
-            rf'[\s，,：:][故即]([{options_concat}])[。\.]?$',
-            rf'[\s，,：:]因此([{options_concat}])[。\.]?$',
-            rf'[是为。]\s?([{options_concat}])[。\.]?$',
-            rf'因此\s?([{options_concat}])[。\.]?$',
-            rf'显然\s?([{options_concat}])[。\.]?$',
-            rf'答案是\s?(\S+)(?:。|$)',
-            rf'答案应该是\s?(\S+)(?:。|$)',
-            rf'答案为\s?(\S+)(?:。|$)',
-            rf'答案是(.*?)[{options_concat}]',
-            rf'答案为(.*?)[{options_concat}]',
-            rf'固选(.*?)[{options_concat}]',
-            rf'答案应该是(.*?)[{options_concat}]',
-            rf'[Tt]he answer is \(?[{options_concat}]\)?',
-            rf'[Tt]he correct answer is [{options_concat}]',
-            rf'[Tt]he correct answer is:\n[{options_concat}]',
-            rf'(\s|^)[{options_concat}][\s。，,\.$]',  # noqa
-            rf'^选项\s?([{options_concat}])',
-            rf'^([{options_concat}])\s?选?项',
-            rf'(\s|^)[{options_concat}][\s。，,：:\.$]',
-            rf'(\s|^)[{options_concat}](\s|$)',
-            rf'[{options_concat}]',
-        ]
-
-        regexes = [re.compile(pattern) for pattern in patterns]
-        for regex in regexes:
-            match = regex.search(text)
-            if match:
-                outputs = match.group(0)
-                for i in options:
-                    if i in outputs:
-                        return i
-        # If no match found, try to find the last capital letter in the text
-        last_capital = ResponseParser.parse_last_capital(text, options)
-        if last_capital:
-            return last_capital
-        return 'No valid option found'
-
-    @staticmethod
-    def parse_first_option(text: str, options: list[str]) -> str:
-        """
-        Find first valid option for text.
-
-        Args:
-            text: The text to parse.
-        """
-        options_pattern = ResponseParser.process_options(options)
-
-        patterns = [
-            rf'[Aa]nswer:\s*({options_pattern})',
-            rf'ANSWER:\s*({options_pattern})',
-            rf'answer is \(?({options_pattern})\)?',
-            rf'[Tt]he correct answer is:\s*({options_pattern})',
-            rf'[Tt]he correct answer is:\n\s*({options_pattern})',
-            rf'[Tt]he correct answer is:\n\n-\s*({options_pattern})',
-            rf'[Tt]he answer might be:\n\n-\s*({options_pattern})',
-            rf'[Tt]he answer is \s*({options_pattern})',
-        ]
-
-        regexes = [re.compile(pattern) for pattern in patterns]
-        for regex in regexes:
-            matches = regex.search(text)
-            if matches:
-                return matches.group(1)
-        # If no match found, try to find the last capital letter in the text
-        last_capital = ResponseParser.parse_last_capital(text, options)
-        if last_capital:
-            return last_capital
-        return 'No valid option found'
-
-    @staticmethod
-    def parse_bracketed_answer(text: str, options: list[str]) -> str:
-        options = ResponseParser.process_options(options)
-        # Match the first occurrence of the options in angle brackets
-        match = re.search(rf'<({options})>', text)
-        if match:
-            return match.group(1)
-        return 'No valid option found'
-
-    @staticmethod
-    def process_options(options: list[str]) -> str:
-        # Escape each option to ensure special characters in options are treated literally
-        escaped_options = [re.escape(option) for option in options]
-        # Join options into a regex pattern separated by '|', to match any of the options
-        options_pattern = '|'.join(escaped_options)
-        return options_pattern
-
-
-if __name__ == '__main__':
-    result = '**Answer: A **Answer: C**'
-    options = ['A', 'B', 'C', 'D']
-    parsed_result = ResponseParser.parse_first_option(result, options)
-    print(f'Parsed result: {parsed_result}')  # Should print 'C'

evalscope/metrics/named_metrics.py

@@ -1,55 +0,0 @@
-from dataclasses import dataclass, field
-from functools import partial
-from typing import Callable, Dict
-
-from evalscope.metrics.metrics import mean, pass_at_k, weighted_mean
-from evalscope.metrics.t2v_metrics import (blip2_score, clip_flant5_score, clip_score, fga_blip2_score, hpsv2_1_score,
-                                           hpsv2_score, image_reward_score, mps_score, pick_score)
-
-
-@dataclass
-class Metric:
-    name: str = 'default_metric'
-    object: Callable = field(default_factory=lambda: mean)
-
-
-class MetricRegistry:
-
-    def __init__(self):
-        self.metrics: Dict[str, Metric] = {}
-
-    def register(self, metric: Metric):
-        self.metrics[metric.name] = metric
-
-    def get(self, name: str) -> Metric:
-        try:
-            return self.metrics[name]
-        except KeyError:
-            raise KeyError(f'Metric {name} not found in the registry. Available metrics: {self.list_metrics()}')
-
-    def list_metrics(self):
-        return list(self.metrics.keys())
-
-
-metric_registry = MetricRegistry()
-
-# Register metrics
-metric_registry.register(Metric(name='AverageAccuracy', object=mean))
-metric_registry.register(Metric(name='WeightedAverageAccuracy', object=weighted_mean))
-metric_registry.register(Metric(name='AverageBLEU', object=mean))
-metric_registry.register(Metric(name='AverageRouge', object=mean))
-metric_registry.register(Metric(name='WeightedAverageBLEU', object=weighted_mean))
-metric_registry.register(Metric(name='AveragePass@1', object=mean))
-for k in range(1, 17):
-    metric_registry.register(Metric(name=f'Pass@{k}', object=partial(pass_at_k, k=k)))
-
-# t2v_metrics
-metric_registry.register(Metric(name='VQAScore', object=clip_flant5_score))
-metric_registry.register(Metric(name='PickScore', object=pick_score))
-metric_registry.register(Metric(name='CLIPScore', object=clip_score))
-metric_registry.register(Metric(name='BLIPv2Score', object=blip2_score))
-metric_registry.register(Metric(name='HPSv2Score', object=hpsv2_score))
-metric_registry.register(Metric(name='HPSv2.1Score', object=hpsv2_1_score))
-metric_registry.register(Metric(name='ImageRewardScore', object=image_reward_score))
-metric_registry.register(Metric(name='FGA_BLIP2Score', object=fga_blip2_score))
-metric_registry.register(Metric(name='MPS', object=mps_score))

evalscope/models/adapters/__init__.py

@@ -1,14 +0,0 @@
-from .base_adapter import BaseModelAdapter, initialize_model_adapter
-from .bfcl_adapter import BFCLAdapter
-from .chat_adapter import ChatGenerationModelAdapter
-from .choice_adapter import ContinuationLogitsModelAdapter, MultiChoiceModelAdapter
-from .custom_adapter import CustomModelAdapter
-from .server_adapter import ServerModelAdapter
-from .t2i_adapter import T2IModelAdapter
-from .tau_bench_adapter import TauBenchAdapter
-
-__all__ = [
-    'initialize_model_adapter', 'BaseModelAdapter', 'ChatGenerationModelAdapter', 'ContinuationLogitsModelAdapter',
-    'MultiChoiceModelAdapter', 'CustomModelAdapter', 'ServerModelAdapter', 'BFCLAdapter', 'T2IModelAdapter',
-    'TauBenchAdapter'
-]

evalscope/models/adapters/base_adapter.py

@@ -1,84 +0,0 @@
-import torch
-from abc import ABC, abstractmethod
-from typing import TYPE_CHECKING, Any, List, Optional, Union
-
-from evalscope.constants import EvalType, OutputType
-from evalscope.utils.logger import get_logger
-from ..custom import CustomModel
-from ..local_model import LocalModel
-
-logger = get_logger()
-
-if TYPE_CHECKING:
-    from evalscope.benchmarks import DataAdapter
-    from evalscope.config import TaskConfig
-
-
-class BaseModelAdapter(ABC):
-
-    def __init__(self, model: Optional[Union[LocalModel, CustomModel]], **kwargs):
-        if model is None:
-            self.model_cfg = kwargs.get('model_cfg', None)
-        elif isinstance(model, LocalModel):
-            self.model = model.model
-            self.model_id = model.model_id
-            self.model_revision = model.model_revision
-            self.device = model.device
-            self.tokenizer = model.tokenizer
-            self.model_cfg = model.model_cfg
-        elif isinstance(model, CustomModel):
-            self.model_cfg = model.config
-        else:
-            raise ValueError(f'Unsupported model type: {type(model)}')
-
-    @abstractmethod
-    @torch.no_grad()
-    def predict(self, *args, **kwargs) -> Any:
-        raise NotImplementedError
-
-
-def initialize_model_adapter(task_cfg: 'TaskConfig', benchmark: 'DataAdapter', base_model: 'LocalModel'):
-    """Initialize the model adapter based on the task configuration."""
-    if task_cfg.eval_type == EvalType.CUSTOM:
-        if not isinstance(task_cfg.model, CustomModel):
-            raise ValueError(f'Expected evalscope.models.custom.CustomModel, but got {type(task_cfg.model)}.')
-        from evalscope.models import CustomModelAdapter
-        return CustomModelAdapter(custom_model=task_cfg.model)
-    else:
-        from ..register import get_model_adapter
-
-        # we need to determine the model adapter class based on the output type
-        model_adapter_cls_str = benchmark.model_adapter
-
-        if task_cfg.eval_type == EvalType.SERVICE or task_cfg.api_url is not None:
-
-            if 'server' not in model_adapter_cls_str:
-                logger.warning(f'Output type {model_adapter_cls_str} is not supported for service evaluation. '
-                               f'Using server model adapter instead.')
-                model_adapter_cls_str = 'server'
-                benchmark.model_adapter = model_adapter_cls_str
-
-            # init server model adapter
-            model_adapter_cls = get_model_adapter(model_adapter_cls_str)
-
-            return model_adapter_cls(
-                api_url=task_cfg.api_url,
-                model_id=task_cfg.model,
-                api_key=task_cfg.api_key,
-                seed=task_cfg.seed,
-                timeout=task_cfg.timeout,
-                stream=task_cfg.stream,
-            )
-        else:
-            if model_adapter_cls_str not in benchmark.output_types:
-                logger.warning(f'Output type {model_adapter_cls_str} is not supported for benchmark {benchmark.name}.'
-                               f'Using {benchmark.output_types[0]} instead.')
-                model_adapter_cls_str = benchmark.output_types[0]
-                benchmark.model_adapter = model_adapter_cls_str
-
-            model_adapter_cls = get_model_adapter(model_adapter_cls_str)
-            return model_adapter_cls(
-                model=base_model,
-                generation_config=task_cfg.generation_config,
-                chat_template=task_cfg.chat_template,
-                task_cfg=task_cfg)

evalscope/models/adapters/bfcl_adapter.py

@@ -1,246 +0,0 @@
-import json
-import time
-import uuid
-from typing import Any, List, Optional, Union
-
-from evalscope.utils.logger import get_logger
-from ..register import register_model_adapter
-from .server_adapter import ServerModelAdapter
-
-logger = get_logger()
-
-
-@register_model_adapter(name='bfcl_server')
-class BFCLAdapter(ServerModelAdapter):
-    """
-    BFCL model adapter to request remote API model and generate results for BFCL evaluation.
-    Support multi-turn and single-turn function calling tasks.
-    """
-
-    def __init__(self, api_url: str, model_id: str, api_key: str = 'EMPTY', **kwargs):
-        """
-        Args:
-            api_url: The URL of the remote API model.
-            model_id: The ID of the remote API model.
-            api_key: The API key of the remote API model.
-        """
-        super().__init__(api_url=api_url, model_id=model_id, api_key=api_key, **kwargs)
-
-    def predict(self, inputs: List[dict], infer_cfg: Optional[dict] = None) -> List[dict]:
-        """
-        Model prediction func. For multi-turn evals, we pass a list[list[message]] to the model
-        where each list is a follow up turn in the conversation
-        each turn is a List[List[Message]]
-
-        Args:
-            inputs (List[dict]): The input data.
-            infer_cfg (dict): Inference configuration.
-
-        Returns:
-            res (List[dict]): The model prediction results.
-        """
-        infer_cfg = infer_cfg or {}
-        results = []
-
-        for input_item in inputs:
-            # This flag decides if we pass tools to the API or try tool calling via prompting
-            # Passing tools to the API means that we rely on the API to manage system prompt specifics
-            # and also expect parsed tool calls in the ChatCompletionMessage object
-            # This is how the is_fc_model=True benchmark is designed to work
-            # On the other hand, we try to manage
-            # tool calling via prompting and parse tool calls in the standard text response
-            # This is how the is_fc_model=False benchmark is designed to work
-            row = input_item.get('messages')
-            is_fc_model = row.get('is_fc_model', False)
-
-            if is_fc_model:
-                response = self.generate_turn_with_tools(row, infer_cfg)
-            else:
-                response = self.generate_turn(row, infer_cfg)
-
-            # wrap response with openai types
-            res_d = {
-                'choices': [{
-                    'index': 0,
-                    'message': {
-                        'content': response,
-                        'role': 'assistant'
-                    }
-                }],
-                'created': time.time(),
-                'model': self.model_id,
-                'object': 'chat.completion',
-                'usage': {
-                    'completion_tokens': 0,
-                    'prompt_tokens': 0,
-                    'total_tokens': 0
-                }
-            }
-            results.append(res_d)
-
-        return results
-
-    def generate_turn(self, row: dict[str, Any], infer_cfg: dict[str, Any]) -> list[str]:
-        from bfcl_eval.constants.default_prompts import (DEFAULT_USER_PROMPT_FOR_ADDITIONAL_FUNCTION_PROMPTING,
-                                                         MAXIMUM_STEP_LIMIT)
-        from bfcl_eval.eval_checker.multi_turn_eval.multi_turn_utils import execute_multi_turn_func_call
-        from bfcl_eval.model_handler.utils import default_decode_execute_prompting
-
-        all_model_responses = []
-        current_messages = []
-        turns = row['turns']
-        for turn_idx, messages in enumerate(turns):
-            n_steps = 0
-            current_responses = []
-            current_messages += messages.copy()
-
-            if str(turn_idx) in row['missing_functions']:
-                assert len(messages) == 0, 'Holdout turn should not have user message.'
-                new_turn = [{
-                    'role':
-                    'user',
-                    'content':
-                    DEFAULT_USER_PROMPT_FOR_ADDITIONAL_FUNCTION_PROMPTING.format(
-                        functions=row['missing_functions'][str(turn_idx)]),
-                }]
-                current_messages += new_turn
-
-            while True:
-                input_item = {
-                    'messages': current_messages,
-                }
-                responses = self.process_single_input(input_item, infer_cfg)
-                result = responses['choices'][0]['message']['content']
-
-                logger.debug(f'Turn:{turn_idx} Step:{n_steps} Result: {result}')
-                current_messages.append({
-                    'role': 'assistant',
-                    'content': result,
-                })
-                current_responses.append(result)
-
-                execute_tools = row.get('should_execute_tool_calls', False)
-                if execute_tools:
-                    try:
-                        tool_calls = default_decode_execute_prompting(result)
-                    except Exception:
-                        tool_calls = None
-
-                    if tool_calls is None:
-                        break
-
-                    tool_outputs, _ = execute_multi_turn_func_call(
-                        tool_calls,
-                        initial_config=row['initial_config'],
-                        involved_classes=row['involved_classes'],
-                        model_name='evaluator_loop',
-                        test_entry_id=row['id'],
-                        long_context=('long_context' in row['test_category'] or 'composite' in row['test_category']),
-                        is_evaL_run=False,
-                    )
-                    # Append tool outputs to the current messages
-                    tool_results = []
-                    for tool_output, tool_call in zip(tool_outputs, tool_calls):
-                        tool_results.append({'role': 'tool', 'name': tool_call, 'content': tool_output})
-                    current_messages.append({
-                        'role': 'user',
-                        'content': repr(tool_results),
-                    })
-                else:
-                    break
-
-                n_steps += 1
-                if n_steps > MAXIMUM_STEP_LIMIT:
-                    logger.error(f'INFERENCE_ERROR: Exceeded max inference steps ({MAXIMUM_STEP_LIMIT})')
-                    break
-
-            all_model_responses.append(current_responses)
-
-        return all_model_responses
-
-    def generate_turn_with_tools(self, row: dict[str, Any], infer_cfg: dict[str, Any]) -> list[str]:
-        from bfcl_eval.constants.default_prompts import (DEFAULT_USER_PROMPT_FOR_ADDITIONAL_FUNCTION_FC,
-                                                         MAXIMUM_STEP_LIMIT)
-        from bfcl_eval.eval_checker.multi_turn_eval.multi_turn_utils import execute_multi_turn_func_call
-        from bfcl_eval.model_handler.utils import convert_to_function_call
-
-        all_model_responses = []
-        current_messages = []
-        turns = row['turns']
-        for turn_idx, messages in enumerate(turns):
-            n_steps = 0
-            current_responses = []
-            current_messages += messages.copy()
-            tools = row['tools']
-
-            if str(turn_idx) in row['missing_functions']:
-                assert len(messages) == 0, 'Holdout turn should not have user message.'
-                # inject new functions on the fly
-                new_tools = row['missing_functions'][str(turn_idx)]
-                for new_tool in new_tools:
-                    tools.append({
-                        'type': 'function',
-                        'function': new_tool[0],
-                    })
-                new_turn = [{
-                    'role': 'user',
-                    'content': DEFAULT_USER_PROMPT_FOR_ADDITIONAL_FUNCTION_FC,
-                }]
-                current_messages += new_turn
-
-            while True:
-                input_item = {
-                    'messages': current_messages,
-                    'tools': tools,
-                }
-                responses = self.process_single_input(input_item, infer_cfg)
-                message = responses['choices'][0]['message']
-
-                current_messages.append(message)
-                if isinstance(message, str):
-                    model_responses = [message]
-                    tool_call_strs = None
-                elif message.get('tool_calls'):
-                    model_responses = [{
-                        tc['function']['name']: tc['function']['arguments']
-                    } for tc in message['tool_calls']]
-                    try:
-                        tool_call_strs = convert_to_function_call(model_responses)
-                    except Exception as e:
-                        logger.error(f'Error converting tool calls to function call strings: {e}')
-                        tool_call_strs = None
-                else:
-                    model_responses = [message['content']]
-                    tool_call_strs = None
-
-                current_responses.extend(model_responses)
-
-                execute_tools = row.get('should_execute_tool_calls', False)
-                if execute_tools and tool_call_strs is not None:
-                    tool_outputs, _ = execute_multi_turn_func_call(
-                        tool_call_strs,
-                        initial_config=row['initial_config'],
-                        involved_classes=row['involved_classes'],
-                        model_name='evaluator_loop',
-                        test_entry_id=row['id'],
-                        long_context=('long_context' in row['test_category'] or 'composite' in row['test_category']),
-                        is_evaL_run=False,
-                    )
-
-                    for tc, tool_output in zip(message['tool_calls'], tool_outputs, strict=False):
-                        current_messages.append({
-                            'role': 'tool',
-                            'tool_call_id': tc['id'],
-                            'content': json.dumps({'response': tool_output}),
-                        })
-                else:
-                    break
-
-                n_steps += 1
-                if n_steps > MAXIMUM_STEP_LIMIT:
-                    logger.error(f'INFERENCE_ERROR: Exceeded max inference steps ({MAXIMUM_STEP_LIMIT})')
-                    break
-
-            all_model_responses.append(current_responses)
-
-        return all_model_responses