evalscope 0.10.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

evalscope/benchmarks/tau_bench/tau_bench/tau_bench_adapter.py

@@ -0,0 +1,168 @@
+from collections import defaultdict
+from typing import Dict, List
+
+from evalscope.api.benchmark import AgentAdapter, BenchmarkMeta
+from evalscope.api.dataset import Sample
+from evalscope.api.dataset.dataset import DatasetDict
+from evalscope.api.dataset.loader import DictDataLoader
+from evalscope.api.messages.chat_message import ChatMessageUser
+from evalscope.api.metric import Score
+from evalscope.api.model import Model, ModelOutput
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils import get_logger
+from evalscope.utils.function_utils import run_once
+from evalscope.utils.import_utils import check_import
+
+logger = get_logger()
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='tau_bench',
+        pretty_name='τ-bench',
+        tags=[Tags.FUNCTION_CALLING, Tags.REASONING, Tags.AGENT],
+        description='A benchmark emulating dynamic conversations between a user (simulated by language models) '
+        'and a language agent provided with domain-specific API tools and policy guidelines. '
+        'Please install it with `pip install git+https://github.com/sierra-research/tau-bench` '
+        'before evaluating and set a user model. [Usage Example](https://evalscope.readthedocs.io/en/latest/third_party/tau_bench.html)',  # noqa: E501
+        dataset_id='https://github.com/sierra-research/tau-bench',
+        subset_list=['airline', 'retail'],
+        aggregation='mean_and_pass_hat_k',
+        eval_split='test',
+        extra_params={
+            'user_model': 'qwen-plus',
+            'api_key': 'EMPTY',
+            'api_base': 'https://dashscope.aliyuncs.com/compatible-mode/v1',
+            'generation_config': {
+                'temperature': 0.0,
+                'max_tokens': 4096,
+            }
+        }
+    )
+)
+class TauBenchAdapter(AgentAdapter):
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+        check_import(
+            'tau_bench',
+            package='git+https://github.com/sierra-research/tau-bench',
+            raise_error=True,
+            feature_name=self.pretty_name
+        )
+
+        # setup user model args
+        self.user_model = self.extra_params.get('user_model', 'qwen-plus')
+        self.api_key = self.extra_params.get('api_key', 'EMPTY')
+        self.api_base = self.extra_params.get('api_base', 'https://dashscope.aliyuncs.com/compatible-mode/v1')
+        self.generation_config = self.extra_params.get('generation_config', {'temperature': 0.0, 'max_tokens': 4096})
+
+        self._patch_env_completion()
+
+    @run_once
+    def _patch_env_completion(self) -> str:
+        from tau_bench.envs.user import LLMUserSimulationEnv
+
+        def new_generate_next_message(self, messages):
+            from evalscope.api.messages import dict_to_chat_message
+            from evalscope.api.model import GenerateConfig, get_model
+            from evalscope.constants import EvalType
+
+            user_server = get_model(
+                model=adapter_instance.user_model,
+                eval_type=EvalType.SERVICE,
+                base_url=adapter_instance.api_base,
+                api_key=adapter_instance.api_key,
+                config=GenerateConfig(**adapter_instance.generation_config)
+            )
+
+            res = user_server.generate(input=[dict_to_chat_message(msg) for msg in messages])
+
+            message = {'role': 'assistant', 'content': res.completion}
+            self.messages.append(message)
+            self.total_cost = 0
+            return res.completion
+
+        # get the current instance of TauBenchAdapter
+        adapter_instance = self
+        LLMUserSimulationEnv.generate_next_message = new_generate_next_message
+
+    def load(self):
+        from tau_bench.envs import get_env
+
+        data_dict = defaultdict(dict)
+        for env_name in self.subset_list:
+            logger.info(f'Loading TauBench environment: {env_name}')
+            env = get_env(
+                env_name=env_name,
+                user_strategy='llm',
+                user_model='dummy',  # Use dummy model to prevent errors
+                user_provider='openai',  # Use dummy provider to prevent errors
+                task_split=self.eval_split,
+            )
+            tasks = []
+            for i in range(len(env.tasks)):
+                tasks.append({
+                    'task_index': i,
+                    'env_name': env_name,
+                })
+            # load dataset
+            dataset = DictDataLoader(
+                dict_list=tasks,
+                sample_fields=self.record_to_sample,
+                limit=self.limit,
+                repeats=self.repeats,
+                shuffle=self.shuffle,
+            ).load()
+
+            data_dict[env_name] = dataset
+
+        test_dataset = DatasetDict(data_dict)
+
+        return test_dataset, None
+
+    def record_to_sample(self, record: Dict) -> Sample:
+        """Convert a data record to a Sample object."""
+        return Sample(
+            input=[ChatMessageUser(content='')],
+            target='',  # Will use the record for evaluation
+            subset_key=record['env_name'],
+            metadata=record  # Store the full record for evaluation
+        )
+
+    def _on_inference(self, model: Model, sample: Sample) -> ModelOutput:
+        from .generation import predict
+        return predict(model, sample)
+
+    def match_score(self, original_prediction: str, filtered_prediction: str, reference: str, task_state) -> Score:
+
+        score = Score(
+            extracted_prediction=filtered_prediction,
+            prediction=original_prediction,
+        )
+
+        try:
+            # Parse the prediction to get the reward
+            task_result = task_state.metadata['task_result']
+            reward = task_result.get('reward', 0.0)
+
+            score.value = {
+                'acc': float(reward),
+            }
+            score.explanation = f'Task completed with reward: {reward}'
+            score.metadata = {
+                'task_result': task_result,
+                'env_name': task_state.metadata.get('env_name', 'unknown'),
+                'task_index': task_state.metadata.get('task_index', -1)
+            }
+            score.main_score_name = 'acc'
+
+        except Exception as e:
+            score.value = {'acc': 0.0}
+            score.explanation = f'Evaluation failed: {str(e)}'
+            score.metadata = {'error': str(e)}
+            score.main_score_name = 'acc'
+
+        return score

evalscope/benchmarks/text2image/evalmuse_adapter.py

@@ -0,0 +1,78 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+from collections import defaultdict
+from typing import List, Optional, Union
+
+from evalscope.api.benchmark import BenchmarkMeta, Text2ImageAdapter
+from evalscope.api.metric.scorer import AggScore, Score
+from evalscope.api.registry import get_metric, register_benchmark
+from evalscope.constants import Tags
+from evalscope.metrics import mean
+from evalscope.utils.function_utils import thread_safe
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='evalmuse',
+        pretty_name='EvalMuse',
+        dataset_id='AI-ModelScope/T2V-Eval-Prompts',
+        description='EvalMuse Text-to-Image Benchmark. Used for evaluating the quality '
+        'and semantic alignment of finely generated images',
+        tags=[Tags.TEXT_TO_IMAGE],
+        subset_list=['EvalMuse'],
+        metric_list=['FGA_BLIP2Score'],
+        few_shot_num=0,
+        train_split=None,
+        eval_split='test',
+    )
+)
+class EvalMuseAdapter(Text2ImageAdapter):
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        assert len(self.metric_list
+                   ) == 1 and self.metric_list[0] == 'FGA_BLIP2Score', 'Only FGA_BLIP2Score is supported for EvalMuse'
+
+    @thread_safe
+    def match_score(self, original_prediction, filtered_prediction, reference, task_state):
+        # Get prediction and prompt from task state
+        image_path = task_state.metadata.get('image_path', original_prediction)
+
+        # Initialize the score object with prediction details
+        score = Score(
+            extracted_prediction=image_path,
+            prediction=image_path,
+        )
+
+        # Calculate scores for each configured metric
+        try:
+            metric_name = self.metric_list[0]
+            metric_cls = get_metric(metric_name)
+            metric_func = metric_cls()  # Initialize with parameters
+            metric_score = metric_func(image_path, task_state.metadata)[0]
+
+            for k, v in metric_score.items():
+                score.value[f'{metric_name}:{k}'] = v.cpu().item()
+        except Exception as e:
+            logger.error(f'Error calculating metric {metric_name}: {e}')
+            score.value[metric_name] = 0
+            score.metadata[metric_name] = f'error: {str(e)}'
+
+        return score
+
+    def aggregate_scores(self, sample_scores) -> List[AggScore]:
+        new_items = defaultdict(list)
+        agg_list = []
+        for sample_score in sample_scores:
+            for metric_name, value in sample_score.score.value.items():
+                metrics_prefix = metric_name.split(':')[0]
+                category = metric_name.rpartition('(')[-1].split(')')[0]
+                category = category.split('-')[0].lower()  # remove the suffix if exists
+                new_items[f'{metrics_prefix}:{category}'].append(value)
+
+        for k, v in new_items.items():
+            agg_list.append(AggScore(metric_name=k, score=mean(v), num=len(v)))
+
+        return agg_list

evalscope/benchmarks/text2image/genai_bench_adapter.py

@@ -0,0 +1,53 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import os
+
+from evalscope.api.benchmark import BenchmarkMeta, Text2ImageAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.messages import ChatMessageUser
+from evalscope.api.registry import get_metric, register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='genai_bench',
+        pretty_name='GenAI-Bench',
+        dataset_id='AI-ModelScope/T2V-Eval-Prompts',
+        description='GenAI-Bench Text-to-Image Benchmark. Includes 1600 prompts for text-to-image task.',
+        tags=[Tags.TEXT_TO_IMAGE],
+        subset_list=['GenAI-Bench-1600'],
+        metric_list=['VQAScore'],
+        few_shot_num=0,
+        train_split=None,
+        eval_split='test',
+    )
+)
+class GenAIBenchAdapter(Text2ImageAdapter):
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    def load_from_disk(self, **kwargs):
+        if os.path.isfile(self.dataset_id):
+            file_name = os.path.basename(self.dataset_id)
+            file_without_ext = os.path.splitext(file_name)[0]
+            self.subset_list = [file_without_ext]
+
+        return super().load_from_disk(use_local_loader=True)
+
+    def record_to_sample(self, record) -> Sample:
+        """Convert a record dictionary to a Sample object."""
+        advanced = record['tags'].get('advanced')
+        return Sample(
+            input=[ChatMessageUser(content=record['prompt'])],
+            metadata={
+                'id': record['id'],
+                'prompt': record['prompt'],
+                'category': 'advanced' if advanced else 'basic',
+                'tags': record.get('tags', []),
+                'image_path': record.get('image_path', ''),  # Optional field for existing image path
+            }
+        )

evalscope/benchmarks/text2image/general_t2i_adapter.py

@@ -0,0 +1,42 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import os
+
+from evalscope.api.benchmark import BenchmarkMeta, Text2ImageAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.messages import ChatMessageSystem, ChatMessageUser
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='general_t2i',
+        dataset_id='general_t2i',
+        description='General Text-to-Image Benchmark',
+        tags=[Tags.TEXT_TO_IMAGE, Tags.CUSTOM],
+        subset_list=['default'],
+        metric_list=['PickScore'],
+        few_shot_num=0,
+        train_split=None,
+        eval_split='test',
+    )
+)
+class GeneralT2IAdapter(Text2ImageAdapter):
+
+    def __init__(self, **kwargs):
+
+        super().__init__(**kwargs)
+
+    def load_from_disk(self, **kwargs):
+        if os.path.isfile(self.dataset_id):
+            file_name = os.path.basename(self.dataset_id)
+            file_without_ext = os.path.splitext(file_name)[0]
+            self.subset_list = [file_without_ext]
+
+        return super().load_from_disk(use_local_loader=True)
+
+    def record_to_sample(self, record):
+        return Sample(input=[ChatMessageUser(content=record['prompt'])], metadata={'image_path': record['image_path']})

evalscope/benchmarks/text2image/hpdv2_adapter.py

@@ -0,0 +1,52 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import os
+
+from evalscope.api.benchmark import BenchmarkMeta, Text2ImageAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.messages import ChatMessageUser
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='hpdv2',
+        pretty_name='HPD-v2',
+        dataset_id='AI-ModelScope/T2V-Eval-Prompts',
+        description='HPDv2 Text-to-Image Benchmark. Evaluation metrics based on human preferences, '
+        'trained on the Human Preference Dataset (HPD v2)',
+        tags=[Tags.TEXT_TO_IMAGE],
+        subset_list=['HPDv2'],
+        metric_list=['HPSv2.1Score'],
+        few_shot_num=0,
+        train_split=None,
+        eval_split='test',
+    )
+)
+class HPDv2Adapter(Text2ImageAdapter):
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    def load_from_disk(self, **kwargs):
+        if os.path.isfile(self.dataset_id):
+            file_name = os.path.basename(self.dataset_id)
+            file_without_ext = os.path.splitext(file_name)[0]
+            self.subset_list = [file_without_ext]
+
+        return super().load_from_disk(use_local_loader=True)
+
+    def record_to_sample(self, record):
+        return Sample(
+            input=[ChatMessageUser(content=record['prompt'])],
+            metadata={
+                'id': record['id'],
+                'prompt': record['prompt'],
+                'category': record.get('tags', {}).get('category', ''),
+                'tags': record.get('tags', {}),
+                'image_path': record.get('image_path', ''),  # Optional field for existing image path
+            }
+        )

evalscope/benchmarks/text2image/tifa_adapter.py

@@ -0,0 +1,27 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+from evalscope.api.benchmark import BenchmarkMeta, Text2ImageAdapter
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='tifa160',
+        pretty_name='TIFA-160',
+        dataset_id='AI-ModelScope/T2V-Eval-Prompts',
+        description='TIFA-160 Text-to-Image Benchmark',
+        tags=[Tags.TEXT_TO_IMAGE],
+        subset_list=['TIFA-160'],
+        metric_list=['PickScore'],
+        few_shot_num=0,
+        train_split=None,
+        eval_split='test',
+    )
+)
+class TIFA_Adapter(Text2ImageAdapter):
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)

evalscope/benchmarks/tool_bench/tool_bench_adapter.py

@@ -0,0 +1,102 @@
+import json
+from typing import Any, Dict
+
+from evalscope.api.benchmark import AgentAdapter, BenchmarkMeta
+from evalscope.api.dataset import Sample
+from evalscope.api.evaluator import TaskState
+from evalscope.api.messages.chat_message import ChatMessage, dict_to_chat_message
+from evalscope.api.metric import Score
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='tool_bench',
+        pretty_name='ToolBench-Static',
+        tags=[Tags.REASONING, Tags.FUNCTION_CALLING],
+        description='ToolBench is a benchmark for evaluating AI models on tool use tasks. '
+        'It includes various subsets such as in-domain and out-of-domain, '
+        'each with its own set of problems that require step-by-step reasoning to arrive at the correct answer. '
+        '[Usage Example](https://evalscope.readthedocs.io/en/latest/third_party/toolbench.html)',
+        dataset_id='AI-ModelScope/ToolBench-Static',
+        subset_list=['in_domain', 'out_of_domain'],
+        metric_list=['Act.EM', 'Plan.EM', 'F1', 'HalluRate', 'Rouge-L'],
+        eval_split='test',
+    )
+)
+class ToolBenchAdapter(AgentAdapter):
+    """
+    ToolBench adapter using the new data processing framework.
+    """
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        """Convert a data record to a Sample object."""
+        messages = record['messages']
+
+        # Process messages and remove the name field, convert function messages
+        processed_messages = []
+        for message in messages:
+            msg_dict = message.copy()
+            if 'name' in msg_dict:
+                del msg_dict['name']
+            if 'role' in msg_dict:
+                if msg_dict['role'] == 'function':
+                    content = json.dumps(msg_dict, ensure_ascii=False)
+                    msg_dict['role'] = 'user'
+                    msg_dict['content'] = content
+
+            # Convert to ChatMessage object
+            chat_msg = dict_to_chat_message(msg_dict)
+            processed_messages.append(chat_msg)
+
+        return Sample(
+            input=processed_messages,
+            target='',  # Store the full record as target for evaluation
+            metadata={
+                'target': record['target'],
+                'tools': record['tools'],
+                'messages': record['messages']
+            }
+        )
+
+    def match_score(
+        self, original_prediction: str, filtered_prediction: str, reference: str, task_state: TaskState
+    ) -> Score:
+        from .utils import calculate_metrics
+
+        score = Score(
+            extracted_prediction=filtered_prediction,
+            prediction=original_prediction,
+        )
+
+        doc = task_state.metadata
+
+        try:
+            data = {
+                'target': doc['target'],
+                'predictions': filtered_prediction,
+                'tools': doc['tools'],
+            }
+            metrics = calculate_metrics(data)
+
+            score.value = metrics
+            score.explanation = f'Metrics: {metrics}'
+            score.metadata = {'target': doc['target'], 'tools': doc['tools'], 'detailed_metrics': metrics}
+            # Set the main score (you can choose the most important metric)
+            score.main_score_name = 'F1'
+
+        except Exception as e:
+            # Handle evaluation errors
+            score.value = {'Act.EM': 0.0, 'Plan.EM': 0.0, 'F1': 0.0, 'HalluRate': 1.0, 'Rouge-L': 0.0}
+            score.explanation = f'Evaluation failed: {str(e)}'
+            score.metadata = {'error': str(e)}
+            score.main_score_name = 'F1'
+
+        return score