evalscope 0.17.1__py3-none-any.whl → 1.0.1__py3-none-any.whl

evalscope/benchmarks/tau_bench/tau_bench_adapter.py

@@ -2,80 +2,90 @@ import importlib
 from collections import defaultdict
 from typing import Dict, List
 
-from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.metrics import Metric, mean, metric_registry
+from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.dataset.dataset import DatasetDict
+from evalscope.api.dataset.loader import DictDataLoader
+from evalscope.api.messages.chat_message import ChatMessageUser
+from evalscope.api.metric import Score
+from evalscope.api.model import Model, ModelOutput
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
 from evalscope.utils import get_logger
+from evalscope.utils.function_utils import run_once
+from evalscope.utils.import_utils import check_import
 
 logger = get_logger()
 
 
-@Benchmark.register(
-    name='tau_bench',
-    pretty_name='τ-bench',
-    tags=['Reasoning', 'Agent', 'Function Calling'],
-    description='A benchmark emulating dynamic conversations between a user (simulated by language models) '
-    'and a language agent provided with domain-specific API tools and policy guidelines. '
-    'Please install it with `pip install git+https://github.com/sierra-research/tau-bench` before evaluating and set a user model. ',  # noqa: E501
-    dataset_id='https://github.com/sierra-research/tau-bench',
-    model_adapter='tau_bench_server',
-    subset_list=['airline', 'retail'],
-    metric_list=['Pass^1'],
-    eval_split='test',
-    extra_params={
-        'user_model': 'qwen-plus',
-        'api_key': 'EMPTY',
-        'api_base': 'https://dashscope.aliyuncs.com/compatible-mode/v1',
-        'generation_config': {
-            'temperature': 0.7,
-            'max_new_tokens': 1024
+@register_benchmark(
+    BenchmarkMeta(
+        name='tau_bench',
+        pretty_name='τ-bench',
+        tags=[Tags.FUNCTION_CALLING, Tags.REASONING],
+        description='A benchmark emulating dynamic conversations between a user (simulated by language models) '
+        'and a language agent provided with domain-specific API tools and policy guidelines. '
+        'Please install it with `pip install git+https://github.com/sierra-research/tau-bench` '
+        'before evaluating and set a user model. [Usage Example](https://evalscope.readthedocs.io/zh-cn/latest/third_party/tau_bench.html)',  # noqa: E501
+        dataset_id='https://github.com/sierra-research/tau-bench',
+        subset_list=['airline', 'retail'],
+        metric_list=['Pass^1'],
+        eval_split='test',
+        extra_params={
+            'user_model': 'qwen-plus',
+            'api_key': 'EMPTY',
+            'api_base': 'https://dashscope.aliyuncs.com/compatible-mode/v1',
+            'generation_config': {
+                'temperature': 0.0,
+                'max_tokens': 4096,
+            }
         }
-    })
-class TauBenchAdapter(DataAdapter):
+    )
+)
+class TauBenchAdapter(DefaultDataAdapter):
 
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
 
-        spec = importlib.util.find_spec('tau_bench')
-        if spec is None:
-            raise ImportError(
-                '`tau_bench` not found, please install it with `pip install git+https://github.com/sierra-research/tau-bench` before evaluating.'  # noqa: E501
-            )
-
-        metric_registry.register(Metric(name='Pass^1', object=mean))
+        check_import('tau_bench', package='git+https://github.com/sierra-research/tau-bench', raise_error=True)
 
         # setup user model args
-        extra_params = kwargs.get('extra_params', {})
-        self.user_model = extra_params.get('user_model', 'qwen-plus')
-        self.api_key = extra_params.get('api_key', 'EMPTY')
-        self.api_base = extra_params.get('api_base', 'https://dashscope.aliyuncs.com/compatible-mode/v1')
-        self.generation_config = extra_params.get('generation_config', {'temperature': 0.7, 'max_new_tokens': 1024})
+        self.user_model = self.extra_params.get('user_model', 'qwen-plus')
+        self.api_key = self.extra_params.get('api_key', 'EMPTY')
+        self.api_base = self.extra_params.get('api_base', 'https://dashscope.aliyuncs.com/compatible-mode/v1')
+        self.generation_config = self.extra_params.get('generation_config', {'temperature': 0.0, 'max_tokens': 4096})
 
         self._patch_env_completion()
 
+    @run_once
     def _patch_env_completion(self) -> str:
         from tau_bench.envs.user import LLMUserSimulationEnv
 
         def new_generate_next_message(self, messages):
-            from evalscope.models import ServerModelAdapter
+            from evalscope.api.messages import dict_to_chat_message
+            from evalscope.api.model import GenerateConfig, get_model
+            from evalscope.constants import EvalType
+
+            user_server = get_model(
+                model=adapter_instance.user_model,
+                eval_type=EvalType.SERVICE,
+                base_url=adapter_instance.api_base,
+                api_key=adapter_instance.api_key,
+                config=GenerateConfig(**adapter_instance.generation_config)
+            )
 
-            user_server = ServerModelAdapter(
-                api_url=adapter_instance.api_base,
-                model_id=adapter_instance.user_model,
-                api_key=adapter_instance.api_key)
-            request_json = user_server.make_request(
-                input_item={'messages': messages}, infer_cfg=adapter_instance.generation_config)
-            res = user_server.send_request(request_json)
+            res = user_server.generate(input=[dict_to_chat_message(msg) for msg in messages])
 
-            message = res['choices'][0]['message']
+            message = {'role': 'assistant', 'content': res.completion}
             self.messages.append(message)
             self.total_cost = 0
-            return message['content']
+            return res.completion
 
         # get the current instance of TauBenchAdapter
         adapter_instance = self
         LLMUserSimulationEnv.generate_next_message = new_generate_next_message
 
-    def load(self, **kwargs):
+    def load(self):
         from tau_bench.envs import get_env
 
         data_dict = defaultdict(dict)
@@ -94,17 +104,61 @@ class TauBenchAdapter(DataAdapter):
                     'task_index': i,
                     'env_name': env_name,
                 })
-            data_dict[env_name][self.eval_split] = tasks
-
-        return data_dict
-
-    def gen_prompt(self, input_d, subset_name, few_shot_list, **kwargs):
-        return self.gen_prompt_data(extra_data=input_d)
-
-    def get_gold_answer(self, input_d):
-        return ''
-
-    def match(self, gold, pred):
-        import json
-        res = json.loads(pred)
-        return res.get('reward', 0.0)
+            # load dataset
+            dataset = DictDataLoader(
+                dict_list=tasks,
+                sample_fields=self.record_to_sample,
+                limit=self.limit,
+                repeats=self.repeats,
+                shuffle=self.shuffle,
+            ).load()
+
+            data_dict[env_name] = dataset
+
+        test_dataset = DatasetDict(data_dict)
+
+        return test_dataset, None
+
+    def record_to_sample(self, record: Dict) -> Sample:
+        """Convert a data record to a Sample object."""
+        return Sample(
+            input=[ChatMessageUser(content='')],
+            target='',  # Will use the record for evaluation
+            subset_key=record['env_name'],
+            metadata=record  # Store the full record for evaluation
+        )
+
+    def _on_inference(self, model: Model, sample: Sample) -> ModelOutput:
+        from .generation import predict
+        return predict(model, sample)
+
+    def match_score(self, original_prediction: str, filtered_prediction: str, reference: str, task_state) -> Score:
+
+        score = Score(
+            extracted_prediction=filtered_prediction,
+            prediction=original_prediction,
+        )
+
+        try:
+            # Parse the prediction to get the reward
+            task_result = task_state.metadata['task_result']
+            reward = task_result.get('reward', 0.0)
+
+            score.value = {
+                'Pass^1': float(reward),
+            }
+            score.explanation = f'Task completed with reward: {reward}'
+            score.metadata = {
+                'task_result': task_result,
+                'env_name': task_state.metadata.get('env_name', 'unknown'),
+                'task_index': task_state.metadata.get('task_index', -1)
+            }
+            score.main_score_name = 'Pass^1'
+
+        except Exception as e:
+            score.value = {'Pass^1': 0.0}
+            score.explanation = f'Evaluation failed: {str(e)}'
+            score.metadata = {'error': str(e)}
+            score.main_score_name = 'Pass^1'
+
+        return score

evalscope/benchmarks/text2image/evalmuse_adapter.py

@@ -0,0 +1,78 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+from collections import defaultdict
+from typing import List, Optional, Union
+
+from evalscope.api.benchmark import BenchmarkMeta, Text2ImageAdapter
+from evalscope.api.metric.scorer import AggScore, Score
+from evalscope.api.registry import get_metric, register_benchmark
+from evalscope.constants import Tags
+from evalscope.metrics import mean
+from evalscope.utils.function_utils import thread_safe
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='evalmuse',
+        pretty_name='EvalMuse',
+        dataset_id='AI-ModelScope/T2V-Eval-Prompts',
+        description='EvalMuse Text-to-Image Benchmark. Used for evaluating the quality '
+        'and semantic alignment of finely generated images',
+        tags=[Tags.TEXT_TO_IMAGE],
+        subset_list=['EvalMuse'],
+        metric_list=['FGA_BLIP2Score'],
+        few_shot_num=0,
+        train_split=None,
+        eval_split='test',
+    )
+)
+class EvalMuseAdapter(Text2ImageAdapter):
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        assert len(self.metric_list
+                   ) == 1 and self.metric_list[0] == 'FGA_BLIP2Score', 'Only FGA_BLIP2Score is supported for EvalMuse'
+
+    @thread_safe
+    def match_score(self, original_prediction, filtered_prediction, reference, task_state):
+        # Get prediction and prompt from task state
+        image_path = task_state.metadata.get('image_path', original_prediction)
+
+        # Initialize the score object with prediction details
+        score = Score(
+            extracted_prediction=image_path,
+            prediction=image_path,
+        )
+
+        # Calculate scores for each configured metric
+        try:
+            metric_name = self.metric_list[0]
+            metric_cls = get_metric(metric_name)
+            metric_func = metric_cls()  # Initialize with parameters
+            metric_score = metric_func(image_path, task_state.metadata)[0]
+
+            for k, v in metric_score.items():
+                score.value[f'{metric_name}:{k}'] = v.cpu().item()
+        except Exception as e:
+            logger.error(f'Error calculating metric {metric_name}: {e}')
+            score.value[metric_name] = 0
+            score.metadata[metric_name] = f'error: {str(e)}'
+
+        return score
+
+    def aggregate_scores(self, sample_scores) -> List[AggScore]:
+        new_items = defaultdict(list)
+        agg_list = []
+        for sample_score in sample_scores:
+            for metric_name, value in sample_score.score.value.items():
+                metrics_prefix = metric_name.split(':')[0]
+                category = metric_name.rpartition('(')[-1].split(')')[0]
+                category = category.split('-')[0].lower()  # remove the suffix if exists
+                new_items[f'{metrics_prefix}:{category}'].append(value)
+
+        for k, v in new_items.items():
+            agg_list.append(AggScore(metric_name=k, score=mean(v), num=len(v)))
+
+        return agg_list

evalscope/benchmarks/text2image/genai_bench_adapter.py

@@ -0,0 +1,53 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import os
+
+from evalscope.api.benchmark import BenchmarkMeta, Text2ImageAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.messages import ChatMessageUser
+from evalscope.api.registry import get_metric, register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='genai_bench',
+        pretty_name='GenAI-Bench',
+        dataset_id='AI-ModelScope/T2V-Eval-Prompts',
+        description='GenAI-Bench Text-to-Image Benchmark. Includes 1600 prompts for text-to-image task.',
+        tags=[Tags.TEXT_TO_IMAGE],
+        subset_list=['GenAI-Bench-1600'],
+        metric_list=['VQAScore'],
+        few_shot_num=0,
+        train_split=None,
+        eval_split='test',
+    )
+)
+class GenAIBenchAdapter(Text2ImageAdapter):
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    def load_from_disk(self, **kwargs):
+        if os.path.isfile(self.dataset_id):
+            file_name = os.path.basename(self.dataset_id)
+            file_without_ext = os.path.splitext(file_name)[0]
+            self.subset_list = [file_without_ext]
+
+        return super().load_from_disk(use_local_loader=True)
+
+    def record_to_sample(self, record) -> Sample:
+        """Convert a record dictionary to a Sample object."""
+        advanced = record['tags'].get('advanced')
+        return Sample(
+            input=[ChatMessageUser(content=record['prompt'])],
+            metadata={
+                'id': record['id'],
+                'prompt': record['prompt'],
+                'category': 'advanced' if advanced else 'basic',
+                'tags': record.get('tags', []),
+                'image_path': record.get('image_path', ''),  # Optional field for existing image path
+            }
+        )

evalscope/benchmarks/text2image/general_t2i_adapter.py

@@ -0,0 +1,42 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import os
+
+from evalscope.api.benchmark import BenchmarkMeta, Text2ImageAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.messages import ChatMessageSystem, ChatMessageUser
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='general_t2i',
+        dataset_id='general_t2i',
+        description='General Text-to-Image Benchmark',
+        tags=[Tags.TEXT_TO_IMAGE, Tags.CUSTOM],
+        subset_list=['default'],
+        metric_list=['PickScore'],
+        few_shot_num=0,
+        train_split=None,
+        eval_split='test',
+    )
+)
+class GeneralT2IAdapter(Text2ImageAdapter):
+
+    def __init__(self, **kwargs):
+
+        super().__init__(**kwargs)
+
+    def load_from_disk(self, **kwargs):
+        if os.path.isfile(self.dataset_id):
+            file_name = os.path.basename(self.dataset_id)
+            file_without_ext = os.path.splitext(file_name)[0]
+            self.subset_list = [file_without_ext]
+
+        return super().load_from_disk(use_local_loader=True)
+
+    def record_to_sample(self, record):
+        return Sample(input=[ChatMessageUser(content=record['prompt'])], metadata={'image_path': record['image_path']})

evalscope/benchmarks/text2image/hpdv2_adapter.py

@@ -0,0 +1,52 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import os
+
+from evalscope.api.benchmark import BenchmarkMeta, Text2ImageAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.messages import ChatMessageUser
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='hpdv2',
+        pretty_name='HPD-v2',
+        dataset_id='AI-ModelScope/T2V-Eval-Prompts',
+        description='HPDv2 Text-to-Image Benchmark. Evaluation metrics based on human preferences, '
+        'trained on the Human Preference Dataset (HPD v2)',
+        tags=[Tags.TEXT_TO_IMAGE],
+        subset_list=['HPDv2'],
+        metric_list=['HPSv2.1Score'],
+        few_shot_num=0,
+        train_split=None,
+        eval_split='test',
+    )
+)
+class HPDv2Adapter(Text2ImageAdapter):
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    def load_from_disk(self, **kwargs):
+        if os.path.isfile(self.dataset_id):
+            file_name = os.path.basename(self.dataset_id)
+            file_without_ext = os.path.splitext(file_name)[0]
+            self.subset_list = [file_without_ext]
+
+        return super().load_from_disk(use_local_loader=True)
+
+    def record_to_sample(self, record):
+        return Sample(
+            input=[ChatMessageUser(content=record['prompt'])],
+            metadata={
+                'id': record['id'],
+                'prompt': record['prompt'],
+                'category': record.get('tags', {}).get('category', ''),
+                'tags': record.get('tags', {}),
+                'image_path': record.get('image_path', ''),  # Optional field for existing image path
+            }
+        )

evalscope/benchmarks/text2image/tifa_adapter.py

@@ -0,0 +1,27 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+from evalscope.api.benchmark import BenchmarkMeta, Text2ImageAdapter
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='tifa160',
+        pretty_name='TIFA-160',
+        dataset_id='AI-ModelScope/T2V-Eval-Prompts',
+        description='TIFA-160 Text-to-Image Benchmark',
+        tags=[Tags.TEXT_TO_IMAGE],
+        subset_list=['TIFA-160'],
+        metric_list=['PickScore'],
+        few_shot_num=0,
+        train_split=None,
+        eval_split='test',
+    )
+)
+class TIFA_Adapter(Text2ImageAdapter):
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)

evalscope/benchmarks/tool_bench/tool_bench_adapter.py

@@ -1,81 +1,102 @@
 import json
-from typing import Dict, List
-
-from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.constants import EvalType, OutputType
-from evalscope.metrics import Metric, mean, metric_registry
-
-
-@Benchmark.register(
-    name='tool_bench',
-    pretty_name='ToolBench-Static',
-    tags=['Reasoning', 'Agent', 'Function Calling'],
-    description='ToolBench is a benchmark for evaluating AI models on tool use tasks. '
-    'It includes various subsets such as in-domain and out-of-domain, '
-    'each with its own set of problems that require step-by-step reasoning to arrive at the correct answer. '
-    '[Usage Example](https://evalscope.readthedocs.io/zh-cn/latest/third_party/toolbench.html)',  # noqa: E501
-    dataset_id='AI-ModelScope/ToolBench-Static',
-    subset_list=['in_domain', 'out_of_domain'],
-    metric_list=['Act.EM', 'Plan.EM', 'F1', 'HalluRate', 'Rouge-L'],
-    few_shot_num=0,
-    train_split=None,
-    eval_split='test',
+from typing import Any, Dict
+
+from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.evaluator import TaskState
+from evalscope.api.messages.chat_message import ChatMessage, dict_to_chat_message
+from evalscope.api.metric import Score
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='tool_bench',
+        pretty_name='ToolBench-Static',
+        tags=[Tags.REASONING, Tags.FUNCTION_CALLING],
+        description='ToolBench is a benchmark for evaluating AI models on tool use tasks. '
+        'It includes various subsets such as in-domain and out-of-domain, '
+        'each with its own set of problems that require step-by-step reasoning to arrive at the correct answer. '
+        '[Usage Example](https://evalscope.readthedocs.io/zh-cn/latest/third_party/toolbench.html)',
+        dataset_id='AI-ModelScope/ToolBench-Static',
+        subset_list=['in_domain', 'out_of_domain'],
+        metric_list=['Act.EM', 'Plan.EM', 'F1', 'HalluRate', 'Rouge-L'],
+        eval_split='test',
+    )
 )
-class ToolBenchAdapter(DataAdapter):
+class ToolBenchAdapter(DefaultDataAdapter):
+    """
+    ToolBench adapter using the new data processing framework.
+    """
 
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
 
-        metric_registry.register(Metric(name='Rouge-L', object=mean))
-        metric_registry.register(Metric(name='Act.EM', object=mean))
-        metric_registry.register(Metric(name='Plan.EM', object=mean))
-        metric_registry.register(Metric(name='F1', object=mean))
-        metric_registry.register(Metric(name='HalluRate', object=mean))
-
-    def gen_prompt(self, input_d: dict, subset_name: str, few_shot_list: list, **kwargs) -> dict:
-        """
-        Generate model prompt from input data.
-        """
-        messages = input_d['messages']
-        # use prepared messages and remove the name field
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        """Convert a data record to a Sample object."""
+        messages = record['messages']
+
+        # Process messages and remove the name field, convert function messages
+        processed_messages = []
         for message in messages:
-            if 'name' in message:
-                del message['name']
-            if 'role' in message:
-                if message['role'] == 'function':
-                    content = json.dumps(message, ensure_ascii=False)
-                    message['role'] = 'user'
-                    message['content'] = content
-        return self.gen_prompt_data(prompt='', messages=messages)
-
-    def get_gold_answer(self, input_d: dict) -> str:
-        """
-        Parse the raw input labels (gold).
-        """
-        return input_d
-
-    def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = EvalType.CHECKPOINT) -> str:
-        """
-        Parse the predicted result and extract proper answer.
-        """
-        return result
-
-    def match(self, gold: dict, pred: str) -> Dict:
-        """
-        Match the gold answer and the predicted answer.
-        """
+            msg_dict = message.copy()
+            if 'name' in msg_dict:
+                del msg_dict['name']
+            if 'role' in msg_dict:
+                if msg_dict['role'] == 'function':
+                    content = json.dumps(msg_dict, ensure_ascii=False)
+                    msg_dict['role'] = 'user'
+                    msg_dict['content'] = content
+
+            # Convert to ChatMessage object
+            chat_msg = dict_to_chat_message(msg_dict)
+            processed_messages.append(chat_msg)
+
+        return Sample(
+            input=processed_messages,
+            target='',  # Store the full record as target for evaluation
+            metadata={
+                'target': record['target'],
+                'tools': record['tools'],
+                'messages': record['messages']
+            }
+        )
+
+    def match_score(
+        self, original_prediction: str, filtered_prediction: str, reference: str, task_state: TaskState
+    ) -> Score:
         from .utils import calculate_metrics
 
-        data = {
-            'target': gold['target'],
-            'predictions': pred,
-            'tools': gold['tools'],
-        }
-        metrics = calculate_metrics(data)
-        return metrics
+        score = Score(
+            extracted_prediction=filtered_prediction,
+            prediction=original_prediction,
+        )
+
+        doc = task_state.metadata
+
+        try:
+            data = {
+                'target': doc['target'],
+                'predictions': filtered_prediction,
+                'tools': doc['tools'],
+            }
+            metrics = calculate_metrics(data)
+
+            score.value = metrics
+            score.explanation = f'Metrics: {metrics}'
+            score.metadata = {'target': doc['target'], 'tools': doc['tools'], 'detailed_metrics': metrics}
+            # Set the main score (you can choose the most important metric)
+            score.main_score_name = 'F1'
 
-    def compute_metric(self, review_res_list: List[dict], **kwargs) -> Dict:
-        # aggregate review results
-        res_dict = super().compute_dict_metric(review_res_list, **kwargs)
+        except Exception as e:
+            # Handle evaluation errors
+            score.value = {'Act.EM': 0.0, 'Plan.EM': 0.0, 'F1': 0.0, 'HalluRate': 1.0, 'Rouge-L': 0.0}
+            score.explanation = f'Evaluation failed: {str(e)}'
+            score.metadata = {'error': str(e)}
+            score.main_score_name = 'F1'
 
-        return super().compute_metric(res_dict, **kwargs)
+        return score