evalscope 1.0.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

evalscope/benchmarks/tau_bench/tau2_bench/tau2_bench_adapter.py

@@ -0,0 +1,146 @@
+import os
+from collections import defaultdict
+from typing import Dict, List
+
+from evalscope.api.benchmark import AgentAdapter, BenchmarkMeta
+from evalscope.api.dataset import Sample
+from evalscope.api.dataset.dataset import DatasetDict
+from evalscope.api.dataset.loader import DictDataLoader
+from evalscope.api.messages.chat_message import ChatMessageUser
+from evalscope.api.metric import Score
+from evalscope.api.model import Model, ModelOutput
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils import get_logger
+from evalscope.utils.function_utils import run_once
+from evalscope.utils.import_utils import check_import
+
+logger = get_logger()
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='tau2_bench',
+        pretty_name='τ²-bench',
+        tags=[Tags.FUNCTION_CALLING, Tags.REASONING, Tags.AGENT],
+        description='τ²-bench (Tau Squared Bench) is an extension and enhancement of the original '
+        'τ-bench (Tau Bench), which is a benchmark designed to evaluate conversational AI agents '
+        'that interact with users through domain-specific API tools and guidelines. '
+        'Please install it with `pip install git+https://github.com/sierra-research/tau2-bench@v0.2.0` '
+        'before evaluating and set a user model. [Usage Example](https://evalscope.readthedocs.io/en/latest/third_party/tau2_bench.html)',  # noqa: E501
+        dataset_id='evalscope/tau2-bench-data',
+        subset_list=['airline', 'retail', 'telecom'],
+        aggregation='mean_and_pass_hat_k',
+        eval_split='test',
+        extra_params={
+            'user_model': 'qwen-plus',
+            'api_key': 'EMPTY',
+            'api_base': 'https://dashscope.aliyuncs.com/compatible-mode/v1',
+            'generation_config': {
+                'temperature': 0.0,
+                'max_tokens': 4096,
+            }
+        }
+    )
+)
+class Tau2BenchAdapter(AgentAdapter):
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+        check_import(
+            'tau2',
+            package='git+https://github.com/sierra-research/tau2-bench@v0.2.0',
+            raise_error=True,
+            feature_name=self.pretty_name
+        )
+
+        # setup user model args
+        self.user_model = self.extra_params.get('user_model', 'qwen-plus')
+        self.api_key = self.extra_params.get('api_key', 'EMPTY')
+        self.api_base = self.extra_params.get('api_base', 'https://dashscope.aliyuncs.com/compatible-mode/v1')
+        self.generation_config = self.extra_params.get('generation_config', {'temperature': 0.0, 'max_tokens': 4096})
+
+    def load(self):
+        # Load dataset
+        dataset_name_or_path = self.dataset_id
+        if os.path.exists(dataset_name_or_path):
+            logger.info(f'Loading dataset from {dataset_name_or_path}')
+            dataset_path = dataset_name_or_path
+        else:
+            from modelscope import dataset_snapshot_download
+            logger.info(f'Loading dataset from modelscope: > dataset_name: {dataset_name_or_path}')
+            dataset_path = dataset_snapshot_download(dataset_name_or_path)
+
+        # Set Tau2 data dir
+        os.environ['TAU2_DATA_DIR'] = dataset_path
+
+        # Load data for each domain
+        from tau2.agent.llm_agent import LLMGTAgent
+        from tau2.registry import registry
+
+        data_dict = defaultdict(dict)
+        for domain_name in self.subset_list:
+            logger.info(f'Loading Tau2-Bench environment: {domain_name}')
+            # Get tasks
+            task_loader = registry.get_tasks_loader(domain_name)
+            tasks = task_loader()
+            tasks = [task for task in tasks if LLMGTAgent.check_valid_task(task)]
+            tasks = [task.model_dump(exclude_unset=True) for task in tasks]
+
+            # load dataset
+            dataset = DictDataLoader(
+                dict_list=tasks,
+                sample_fields=self.record_to_sample,
+                limit=self.limit,
+                repeats=self.repeats,
+                shuffle=self.shuffle,
+            ).load()
+
+            data_dict[domain_name] = dataset
+
+        test_dataset = DatasetDict(data_dict)
+
+        return test_dataset, None
+
+    def record_to_sample(self, record: Dict) -> Sample:
+        """Convert a data record to a Sample object."""
+        return Sample(
+            input=[ChatMessageUser(content=record['description']['purpose'] or '')],
+            target='',  # Will use the record for evaluation
+            subset_key=record['user_scenario']['instructions']['domain'],
+            metadata=record  # Store the full record for evaluation
+        )
+
+    def _on_inference(self, model: Model, sample: Sample) -> ModelOutput:
+        from .generation import predict
+        return predict(model, sample, adapter_instance=self)
+
+    def match_score(self, original_prediction: str, filtered_prediction: str, reference: str, task_state) -> Score:
+
+        score = Score(
+            extracted_prediction=filtered_prediction,
+            prediction=original_prediction,
+        )
+
+        try:
+            # Parse the prediction to get the reward
+            task_result = task_state.metadata['task_result']
+            reward = task_result['reward']
+
+            score.value = {
+                'acc': float(reward),
+            }
+            score.explanation = f'Task completed with reward: {reward}'
+            score.metadata = {
+                'task_result': task_result,
+            }
+            score.main_score_name = 'acc'
+
+        except Exception as e:
+            score.value = {'acc': 0.0}
+            score.explanation = f'Evaluation failed: {str(e)}'
+            score.metadata = {'error': str(e)}
+            score.main_score_name = 'acc'
+
+        return score

evalscope/benchmarks/tau_bench/{generation.py → tau_bench/generation.py}

@@ -45,7 +45,7 @@ def _patch_agent_solve(model: Model):
                 input=[dict_to_chat_message(msg) for msg in messages],
                 tools=[ToolInfo.model_validate(tool['function']) for tool in self.tools_info]
             )
-            oai_res = openai_chat_choices(res.choices)
+            oai_res = openai_chat_choices(res.choices, include_reasoning=False)
 
             next_message = oai_res[0].message.model_dump(exclude_none=True)
 

evalscope/benchmarks/tau_bench/{tau_bench_adapter.py → tau_bench/tau_bench_adapter.py}

@@ -1,8 +1,7 @@
-import importlib
 from collections import defaultdict
 from typing import Dict, List
 
-from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter
+from evalscope.api.benchmark import AgentAdapter, BenchmarkMeta
 from evalscope.api.dataset import Sample
 from evalscope.api.dataset.dataset import DatasetDict
 from evalscope.api.dataset.loader import DictDataLoader
@@ -13,6 +12,7 @@ from evalscope.api.registry import register_benchmark
 from evalscope.constants import Tags
 from evalscope.utils import get_logger
 from evalscope.utils.function_utils import run_once
+from evalscope.utils.import_utils import check_import
 
 logger = get_logger()
 
@@ -21,47 +21,43 @@ logger = get_logger()
     BenchmarkMeta(
         name='tau_bench',
         pretty_name='τ-bench',
-        tags=[Tags.FUNCTION_CALLING, Tags.REASONING],
+        tags=[Tags.FUNCTION_CALLING, Tags.REASONING, Tags.AGENT],
         description='A benchmark emulating dynamic conversations between a user (simulated by language models) '
         'and a language agent provided with domain-specific API tools and policy guidelines. '
         'Please install it with `pip install git+https://github.com/sierra-research/tau-bench` '
-        'before evaluating and set a user model. [Usage Example](https://evalscope.readthedocs.io/zh-cn/latest/third_party/tau_bench.html)',  # noqa: E501
+        'before evaluating and set a user model. [Usage Example](https://evalscope.readthedocs.io/en/latest/third_party/tau_bench.html)',  # noqa: E501
         dataset_id='https://github.com/sierra-research/tau-bench',
         subset_list=['airline', 'retail'],
-        metric_list=['Pass^1'],
+        aggregation='mean_and_pass_hat_k',
         eval_split='test',
         extra_params={
             'user_model': 'qwen-plus',
             'api_key': 'EMPTY',
             'api_base': 'https://dashscope.aliyuncs.com/compatible-mode/v1',
             'generation_config': {
-                'temperature': 0.7,
-                'max_new_tokens': 1024
+                'temperature': 0.0,
+                'max_tokens': 4096,
             }
         }
     )
 )
-class TauBenchAdapter(DefaultDataAdapter):
+class TauBenchAdapter(AgentAdapter):
 
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
 
-        spec = importlib.util.find_spec('tau_bench')
-        if spec is None:
-            raise ImportError(
-                '`tau_bench` not found, please install it with `pip install git+https://github.com/sierra-research/tau-bench` before evaluating.'  # noqa: E501
-            )
+        check_import(
+            'tau_bench',
+            package='git+https://github.com/sierra-research/tau-bench',
+            raise_error=True,
+            feature_name=self.pretty_name
+        )
 
         # setup user model args
         self.user_model = self.extra_params.get('user_model', 'qwen-plus')
         self.api_key = self.extra_params.get('api_key', 'EMPTY')
         self.api_base = self.extra_params.get('api_base', 'https://dashscope.aliyuncs.com/compatible-mode/v1')
-        self.generation_config = self.extra_params.get(
-            'generation_config', {
-                'temperature': 0.7,
-                'max_new_tokens': 1024
-            }
-        )
+        self.generation_config = self.extra_params.get('generation_config', {'temperature': 0.0, 'max_tokens': 4096})
 
         self._patch_env_completion()
 
@@ -84,10 +80,10 @@ class TauBenchAdapter(DefaultDataAdapter):
 
             res = user_server.generate(input=[dict_to_chat_message(msg) for msg in messages])
 
-            message = res.message.model_dump(exclude_none=True)
+            message = {'role': 'assistant', 'content': res.completion}
             self.messages.append(message)
             self.total_cost = 0
-            return message['content']
+            return res.completion
 
         # get the current instance of TauBenchAdapter
         adapter_instance = self
@@ -114,7 +110,11 @@ class TauBenchAdapter(DefaultDataAdapter):
                 })
             # load dataset
             dataset = DictDataLoader(
-                dict_list=tasks, sample_fields=self.record_to_sample, limit=self.limit, repeats=self.repeats
+                dict_list=tasks,
+                sample_fields=self.record_to_sample,
+                limit=self.limit,
+                repeats=self.repeats,
+                shuffle=self.shuffle,
             ).load()
 
             data_dict[env_name] = dataset
@@ -145,24 +145,24 @@ class TauBenchAdapter(DefaultDataAdapter):
 
         try:
             # Parse the prediction to get the reward
-            res = task_state.metadata
-            reward = res.get('reward', 0.0)
+            task_result = task_state.metadata['task_result']
+            reward = task_result.get('reward', 0.0)
 
             score.value = {
-                'Pass^1': float(reward),
+                'acc': float(reward),
             }
             score.explanation = f'Task completed with reward: {reward}'
             score.metadata = {
-                'task_result': res,
+                'task_result': task_result,
                 'env_name': task_state.metadata.get('env_name', 'unknown'),
                 'task_index': task_state.metadata.get('task_index', -1)
             }
-            score.main_score_name = 'Pass^1'
+            score.main_score_name = 'acc'
 
         except Exception as e:
-            score.value = {'Pass^1': 0.0}
+            score.value = {'acc': 0.0}
             score.explanation = f'Evaluation failed: {str(e)}'
             score.metadata = {'error': str(e)}
-            score.main_score_name = 'Pass^1'
+            score.main_score_name = 'acc'
 
         return score

evalscope/benchmarks/{aigc/t2i → text2image}/evalmuse_adapter.py

@@ -16,8 +16,10 @@ logger = get_logger()
 @register_benchmark(
     BenchmarkMeta(
         name='evalmuse',
+        pretty_name='EvalMuse',
         dataset_id='AI-ModelScope/T2V-Eval-Prompts',
-        description='EvalMuse Text-to-Image Benchmark',
+        description='EvalMuse Text-to-Image Benchmark. Used for evaluating the quality '
+        'and semantic alignment of finely generated images',
         tags=[Tags.TEXT_TO_IMAGE],
         subset_list=['EvalMuse'],
         metric_list=['FGA_BLIP2Score'],

evalscope/benchmarks/{aigc/t2i → text2image}/genai_bench_adapter.py

@@ -4,7 +4,6 @@ import os
 from evalscope.api.benchmark import BenchmarkMeta, Text2ImageAdapter
 from evalscope.api.dataset import Sample
 from evalscope.api.messages import ChatMessageUser
-from evalscope.api.metric.scorer import Score
 from evalscope.api.registry import get_metric, register_benchmark
 from evalscope.constants import Tags
 from evalscope.utils.logger import get_logger
@@ -15,8 +14,9 @@ logger = get_logger()
 @register_benchmark(
     BenchmarkMeta(
         name='genai_bench',
+        pretty_name='GenAI-Bench',
         dataset_id='AI-ModelScope/T2V-Eval-Prompts',
-        description='GenAI-Bench Text-to-Image Benchmark',
+        description='GenAI-Bench Text-to-Image Benchmark. Includes 1600 prompts for text-to-image task.',
         tags=[Tags.TEXT_TO_IMAGE],
         subset_list=['GenAI-Bench-1600'],
         metric_list=['VQAScore'],

evalscope/benchmarks/{aigc/t2i → text2image}/general_t2i_adapter.py

@@ -16,7 +16,7 @@ logger = get_logger()
         name='general_t2i',
         dataset_id='general_t2i',
         description='General Text-to-Image Benchmark',
-        tags=[Tags.TEXT_TO_IMAGE],
+        tags=[Tags.TEXT_TO_IMAGE, Tags.CUSTOM],
         subset_list=['default'],
         metric_list=['PickScore'],
         few_shot_num=0,

evalscope/benchmarks/{aigc/t2i → text2image}/hpdv2_adapter.py

@@ -14,8 +14,10 @@ logger = get_logger()
 @register_benchmark(
     BenchmarkMeta(
         name='hpdv2',
+        pretty_name='HPD-v2',
         dataset_id='AI-ModelScope/T2V-Eval-Prompts',
-        description='HPDv2 Text-to-Image Benchmark',
+        description='HPDv2 Text-to-Image Benchmark. Evaluation metrics based on human preferences, '
+        'trained on the Human Preference Dataset (HPD v2)',
         tags=[Tags.TEXT_TO_IMAGE],
         subset_list=['HPDv2'],
         metric_list=['HPSv2.1Score'],
@@ -41,7 +43,10 @@ class HPDv2Adapter(Text2ImageAdapter):
         return Sample(
             input=[ChatMessageUser(content=record['prompt'])],
             metadata={
+                'id': record['id'],
+                'prompt': record['prompt'],
                 'category': record.get('tags', {}).get('category', ''),
-                'tags': record.get('tags', {})
+                'tags': record.get('tags', {}),
+                'image_path': record.get('image_path', ''),  # Optional field for existing image path
             }
         )

evalscope/benchmarks/{aigc/t2i → text2image}/tifa_adapter.py

@@ -10,6 +10,7 @@ logger = get_logger()
 @register_benchmark(
     BenchmarkMeta(
         name='tifa160',
+        pretty_name='TIFA-160',
         dataset_id='AI-ModelScope/T2V-Eval-Prompts',
         description='TIFA-160 Text-to-Image Benchmark',
         tags=[Tags.TEXT_TO_IMAGE],

evalscope/benchmarks/tool_bench/tool_bench_adapter.py

@@ -1,7 +1,7 @@
 import json
 from typing import Any, Dict
 
-from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter
+from evalscope.api.benchmark import AgentAdapter, BenchmarkMeta
 from evalscope.api.dataset import Sample
 from evalscope.api.evaluator import TaskState
 from evalscope.api.messages.chat_message import ChatMessage, dict_to_chat_message
@@ -21,14 +21,14 @@ logger = get_logger()
         description='ToolBench is a benchmark for evaluating AI models on tool use tasks. '
         'It includes various subsets such as in-domain and out-of-domain, '
         'each with its own set of problems that require step-by-step reasoning to arrive at the correct answer. '
-        '[Usage Example](https://evalscope.readthedocs.io/zh-cn/latest/third_party/toolbench.html)',
+        '[Usage Example](https://evalscope.readthedocs.io/en/latest/third_party/toolbench.html)',
         dataset_id='AI-ModelScope/ToolBench-Static',
         subset_list=['in_domain', 'out_of_domain'],
         metric_list=['Act.EM', 'Plan.EM', 'F1', 'HalluRate', 'Rouge-L'],
         eval_split='test',
     )
 )
-class ToolBenchAdapter(DefaultDataAdapter):
+class ToolBenchAdapter(AgentAdapter):
     """
     ToolBench adapter using the new data processing framework.
     """

evalscope/benchmarks/truthful_qa/truthful_qa_adapter.py

@@ -37,6 +37,7 @@ TRUTHFUL_QA_PROMPT = (
         dataset_id='evalscope/truthful_qa',
         metric_list=['multi_choice_acc'],
         subset_list=['multiple_choice'],
+        shuffle_choices=True,
         few_shot_num=0,
         train_split=None,
         eval_split='validation',
@@ -55,8 +56,6 @@ class TruthfulQaAdapter(MultiChoiceAdapter):
 
         super().__init__(**kwargs)
 
-        self.shuffle_choices = True
-
         self.multiple_correct = self.extra_params.get('multiple_correct', False)
         if self.multiple_correct:
             self.prompt_template = MultipleChoiceTemplate.MULTIPLE_ANSWER

evalscope/benchmarks/visu_logic/visu_logic_adapter.py

@@ -0,0 +1,75 @@
+# flake8: noqa: E501
+from typing import Any, Dict, List
+
+from evalscope.api.benchmark import BenchmarkMeta, VisionLanguageAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.evaluator import TaskState
+from evalscope.api.messages import ChatMessageUser, Content, ContentImage, ContentText
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.io_utils import bytes_to_base64
+from evalscope.utils.logger import get_logger
+from evalscope.utils.multi_choices import parse_answers
+
+logger = get_logger()
+
+MULT_CHOICE_PROMPT = """
+Answer the following multiple choice question. The last line of your response should be of the following format: 'ANSWER: $LETTER' (without quotes) where LETTER is one of A, B, C, D. Think step by step before answering.
+
+{question}
+"""
+
+SUBSET_LIST = [
+    'Quantitative Reasoning', 'Other', 'Positional Reasoning', 'Stylistic Reasoning', 'Spatial Reasoning',
+    'Attribute Reasoning'
+]
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='visulogic',
+        pretty_name='VisuLogic',
+        dataset_id='evalscope/VisuLogic',
+        tags=[Tags.MATH, Tags.REASONING, Tags.MULTIPLE_CHOICE, Tags.MULTI_MODAL],
+        description=
+        'VisuLogic is a benchmark aimed at evaluating the visual reasoning capabilities of Multi-modal Large Language Models (MLLMs), independent of textual reasoning processes. It features carefully constructed visual reasoning tasks spanning multiple categories, divided into six types based on required reasoning skills (e.g., Quantitative Reasoning, which involves understanding and deducing changes in the quantity of elements in images). Unlike existing benchmarks, VisuLogic is a challenging visual reasoning benchmark that is inherently difficult to articulate using language, providing a more rigorous evaluation of the visual reasoning capabilities of MLLMs.',
+        subset_list=SUBSET_LIST,
+        metric_list=['acc'],
+        eval_split='test',
+        prompt_template=MULT_CHOICE_PROMPT,
+    )
+)
+class VisuLogicAdapter(VisionLanguageAdapter):
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.reformat_subset = True
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        question = record.get('question', '')
+        content_list: List[Content] = []
+        prompt_text = self.prompt_template.format(question=question).strip()
+        content_list.append(ContentText(text=prompt_text))
+
+        image = record.get('image')
+        if image and isinstance(image, dict):
+            image_bytes = image.get('bytes')
+            if image_bytes:
+                image_base64 = bytes_to_base64(image_bytes, format='png', add_header=True)
+                content_list.append(ContentImage(image=image_base64))
+
+        metadata = {
+            'id': record['id'],
+        }
+
+        return Sample(
+            input=[ChatMessageUser(content=content_list)],
+            target=record['label'],
+            choices=['A', 'B', 'C', 'D'],
+            subset_key=record['tag'],
+            metadata=metadata,
+        )
+
+    def extract_answer(self, prediction: str, task_state: TaskState) -> str:
+        answers = parse_answers(task_state)
+        return ''.join(sorted(list(answers)))