evalscope 0.17.1__py3-none-any.whl → 1.0.0__py3-none-any.whl

evalscope/benchmarks/tool_bench/tool_bench_adapter.py

@@ -1,81 +1,102 @@
 import json
-from typing import Dict, List
-
-from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.constants import EvalType, OutputType
-from evalscope.metrics import Metric, mean, metric_registry
-
-
-@Benchmark.register(
-    name='tool_bench',
-    pretty_name='ToolBench-Static',
-    tags=['Reasoning', 'Agent', 'Function Calling'],
-    description='ToolBench is a benchmark for evaluating AI models on tool use tasks. '
-    'It includes various subsets such as in-domain and out-of-domain, '
-    'each with its own set of problems that require step-by-step reasoning to arrive at the correct answer. '
-    '[Usage Example](https://evalscope.readthedocs.io/zh-cn/latest/third_party/toolbench.html)',  # noqa: E501
-    dataset_id='AI-ModelScope/ToolBench-Static',
-    subset_list=['in_domain', 'out_of_domain'],
-    metric_list=['Act.EM', 'Plan.EM', 'F1', 'HalluRate', 'Rouge-L'],
-    few_shot_num=0,
-    train_split=None,
-    eval_split='test',
+from typing import Any, Dict
+
+from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.evaluator import TaskState
+from evalscope.api.messages.chat_message import ChatMessage, dict_to_chat_message
+from evalscope.api.metric import Score
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='tool_bench',
+        pretty_name='ToolBench-Static',
+        tags=[Tags.REASONING, Tags.FUNCTION_CALLING],
+        description='ToolBench is a benchmark for evaluating AI models on tool use tasks. '
+        'It includes various subsets such as in-domain and out-of-domain, '
+        'each with its own set of problems that require step-by-step reasoning to arrive at the correct answer. '
+        '[Usage Example](https://evalscope.readthedocs.io/zh-cn/latest/third_party/toolbench.html)',
+        dataset_id='AI-ModelScope/ToolBench-Static',
+        subset_list=['in_domain', 'out_of_domain'],
+        metric_list=['Act.EM', 'Plan.EM', 'F1', 'HalluRate', 'Rouge-L'],
+        eval_split='test',
+    )
 )
-class ToolBenchAdapter(DataAdapter):
+class ToolBenchAdapter(DefaultDataAdapter):
+    """
+    ToolBench adapter using the new data processing framework.
+    """
 
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
 
-        metric_registry.register(Metric(name='Rouge-L', object=mean))
-        metric_registry.register(Metric(name='Act.EM', object=mean))
-        metric_registry.register(Metric(name='Plan.EM', object=mean))
-        metric_registry.register(Metric(name='F1', object=mean))
-        metric_registry.register(Metric(name='HalluRate', object=mean))
-
-    def gen_prompt(self, input_d: dict, subset_name: str, few_shot_list: list, **kwargs) -> dict:
-        """
-        Generate model prompt from input data.
-        """
-        messages = input_d['messages']
-        # use prepared messages and remove the name field
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        """Convert a data record to a Sample object."""
+        messages = record['messages']
+
+        # Process messages and remove the name field, convert function messages
+        processed_messages = []
         for message in messages:
-            if 'name' in message:
-                del message['name']
-            if 'role' in message:
-                if message['role'] == 'function':
-                    content = json.dumps(message, ensure_ascii=False)
-                    message['role'] = 'user'
-                    message['content'] = content
-        return self.gen_prompt_data(prompt='', messages=messages)
-
-    def get_gold_answer(self, input_d: dict) -> str:
-        """
-        Parse the raw input labels (gold).
-        """
-        return input_d
-
-    def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = EvalType.CHECKPOINT) -> str:
-        """
-        Parse the predicted result and extract proper answer.
-        """
-        return result
-
-    def match(self, gold: dict, pred: str) -> Dict:
-        """
-        Match the gold answer and the predicted answer.
-        """
+            msg_dict = message.copy()
+            if 'name' in msg_dict:
+                del msg_dict['name']
+            if 'role' in msg_dict:
+                if msg_dict['role'] == 'function':
+                    content = json.dumps(msg_dict, ensure_ascii=False)
+                    msg_dict['role'] = 'user'
+                    msg_dict['content'] = content
+
+            # Convert to ChatMessage object
+            chat_msg = dict_to_chat_message(msg_dict)
+            processed_messages.append(chat_msg)
+
+        return Sample(
+            input=processed_messages,
+            target='',  # Store the full record as target for evaluation
+            metadata={
+                'target': record['target'],
+                'tools': record['tools'],
+                'messages': record['messages']
+            }
+        )
+
+    def match_score(
+        self, original_prediction: str, filtered_prediction: str, reference: str, task_state: TaskState
+    ) -> Score:
         from .utils import calculate_metrics
 
-        data = {
-            'target': gold['target'],
-            'predictions': pred,
-            'tools': gold['tools'],
-        }
-        metrics = calculate_metrics(data)
-        return metrics
+        score = Score(
+            extracted_prediction=filtered_prediction,
+            prediction=original_prediction,
+        )
+
+        doc = task_state.metadata
+
+        try:
+            data = {
+                'target': doc['target'],
+                'predictions': filtered_prediction,
+                'tools': doc['tools'],
+            }
+            metrics = calculate_metrics(data)
+
+            score.value = metrics
+            score.explanation = f'Metrics: {metrics}'
+            score.metadata = {'target': doc['target'], 'tools': doc['tools'], 'detailed_metrics': metrics}
+            # Set the main score (you can choose the most important metric)
+            score.main_score_name = 'F1'
 
-    def compute_metric(self, review_res_list: List[dict], **kwargs) -> Dict:
-        # aggregate review results
-        res_dict = super().compute_dict_metric(review_res_list, **kwargs)
+        except Exception as e:
+            # Handle evaluation errors
+            score.value = {'Act.EM': 0.0, 'Plan.EM': 0.0, 'F1': 0.0, 'HalluRate': 1.0, 'Rouge-L': 0.0}
+            score.explanation = f'Evaluation failed: {str(e)}'
+            score.metadata = {'error': str(e)}
+            score.main_score_name = 'F1'
 
-        return super().compute_metric(res_dict, **kwargs)
+        return score

evalscope/benchmarks/trivia_qa/trivia_qa_adapter.py

@@ -1,142 +1,74 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 # Copyright (c) EleutherAI Inc, and its affiliates.
-import csv
-import os
 
-from evalscope.benchmarks import Benchmark
-from evalscope.benchmarks.data_adapter import DataAdapter
-from evalscope.constants import EvalType, OutputType
-from evalscope.utils import get_logger
+from typing import Any, Dict
 
-# flake8: noqa
+from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.evaluator import TaskState
+from evalscope.api.metric import Score
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.logger import get_logger
 
 logger = get_logger()
 
+PROMPT_TEMPLATE = """
+Read the content and answer the following question.
 
-@Benchmark.register(
-    name='trivia_qa',
-    pretty_name='TriviaQA',
-    tags=['QA', 'Reading Comprehension'],
-    description=
-    'TriviaQA is a large-scale reading comprehension dataset consisting of question-answer pairs collected from trivia websites. It includes questions with multiple possible answers, making it suitable for evaluating the ability of models to understand and generate answers based on context.',  # noqa: E501
-    dataset_id='modelscope/trivia_qa',
-    subset_list=['default'],
-    metric_list=['AverageAccuracy'],
-    few_shot_num=5,
-    train_split='dev',
-    eval_split='test',
-)
-class TriviaQaAdapter(DataAdapter):
-
-    def __init__(self, **kwargs):
-
-        super().__init__(**kwargs)
+Content: {content}
 
-    def load_from_disk(self, dataset_name_or_path, subset_list, work_dir, **kwargs) -> dict:
-        data_dict = {}
-        for subset_name in subset_list:
-            data_dict[subset_name] = {}
-            for split in [self.train_split, self.eval_split]:
-                if os.path.exists(dataset_name_or_path):
-                    file_path = os.path.join(dataset_name_or_path, f'trivia-{split}.qa.csv')
-                else:
-                    file_path = os.path.join(work_dir, dataset_name_or_path, f'trivia-{split}.qa.csv')
-                if os.path.exists(file_path):
-                    with open(file_path, 'r', encoding='utf-8') as f:
-                        reader = csv.reader(f, delimiter='\t')
-                        split_data = []
-                        for row in reader:
-                            assert len(row) == 2
-                            question = row[0]
-                            answers = eval(row[1])
-                            split_data.append({
-                                'input': [{
-                                    'role': 'system',
-                                    'content': 'Follow the given examples and answer the question.'
-                                }, {
-                                    'role': 'user',
-                                    'content': question
-                                }],
-                                'ideal':
-                                answers
-                            })
-                        data_dict[subset_name][split] = split_data
+Question: {question}
 
-        return data_dict
+Keep your The last line of your response should be of the form "ANSWER: $ANSWER" (without quotes) where $ANSWER is the answer to the problem.
+""".lstrip()  # noqa: E501
 
-    def gen_prompt(self, input_d: dict, subset_name: str, few_shot_list: list, **kwargs) -> dict:
-        """
-        Generate model prompt from raw input, unify the prompt format for TriviaQA benchmark.
 
-        Args:
-            input_d (dict): The raw input. A single data format of the TriviaQA:
-
-            {
-                "input": [
-                    {"role": "system", "content": "Follow the given examples and answer the question."},
-                    {"role": "user", "content": "Which Lloyd Webber musical premiered in the US on 10th December 1993?"}
-                ],
-                "ideal": [
-                    "Sunset Blvd",
-                    "West Sunset Boulevard",
-                    "Sunset Boulevard",
-                    "Sunset Bulevard",
-                    "Sunset Blvd.",
-                    "sunset boulevard",
-                    "sunset bulevard",
-                    "west sunset boulevard",
-                    "sunset blvd"
-                ]
+@register_benchmark(
+    BenchmarkMeta(
+        name='trivia_qa',
+        pretty_name='TriviaQA',
+        dataset_id='evalscope/trivia_qa',
+        tags=[Tags.QA, Tags.READING_COMPREHENSION],
+        description=
+        'TriviaQA is a large-scale reading comprehension dataset consisting of question-answer pairs collected from trivia websites. It includes questions with multiple possible answers, making it suitable for evaluating the ability of models to understand and generate answers based on context.',  # noqa: E501
+        subset_list=['rc.wikipedia'],
+        few_shot_num=0,
+        train_split=None,
+        eval_split='validation',
+        metric_list=[{
+            'acc': {
+                'allow_inclusion': True
             }
+        }],
+        prompt_template=PROMPT_TEMPLATE,
+    )
+)
+class TriviaQaAdapter(DefaultDataAdapter):
 
-        Returns:
-            {'data': [(context, continuation), ...]}
-        """
-
-        def get_sys_prompt(inp: dict) -> str:
-            return inp['input'][0]['content']
-
-        if self.few_shot_num > 0:
-            sys_prompt = get_sys_prompt(input_d)
-        else:
-            sys_prompt = None
-        few_shot_prompts = [self._generate_prompt(input_d=sample, include_answer=True) for sample in few_shot_list]
-        context = '\n'.join(few_shot_prompts) + '\n'
-        context += self._generate_prompt(input_d=input_d, include_answer=False)
-        full_prompt = context
-
-        return self.gen_prompt_data(full_prompt, system_prompt=sys_prompt)
-
-    def get_gold_answer(self, input_d: dict) -> list:
-        # Get the gold choice
-        ans: list = input_d.get('ideal', [])
-        return ans
-
-    def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = EvalType.CHECKPOINT) -> str:
-        """
-        Parse the model output to get the answer.
-
-        Args:
-            result: Predicted answer from the model. A list of loglikelihood values for inputs pairs.
-            raw_input_d: The raw input. A single data format of the TriviaQA:
-            eval_type: The type of evaluation, e.g. 'checkpoint' or 'service' or 'custom'.
-
-        Returns:
-            The predicted answer.
-        """
-        return result
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
 
-    def match(self, gold: list, pred: str) -> float:
-        lower_pred = pred.lower()
-        gold = [g.lower() for g in gold]
-        is_correct = any([cand in lower_pred for cand in gold])
-        return 1 if is_correct else 0
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        question = record['question']
+        answers = record['answer']['aliases'] + record['answer']['normalized_aliases']
+        content = record['entity_pages']['wiki_context']
+        return Sample(
+            input=question, target=answers, metadata={
+                'question_id': record['question_id'],
+                'content': content
+            }
+        )
 
-    @classmethod
-    def _generate_prompt(cls, input_d: dict, include_answer=True) -> str:
+    def format_prompt_template(self, sample):
+        return self.prompt_template.format(content=sample.metadata['content'], question=sample.input)
 
-        example: str = f"Question: {input_d['input'][1]['content']}\nAnswer:"
-        if include_answer:
-            example += f" {input_d['ideal'][0]}\n\n"
+    def extract_answer(self, prediction: str, task_state: TaskState):
+        # use regex to extract the answer from the prediction
+        import re
 
-        return example
+        pattern = r'ANSWER:\s*(.*)'
+        match = re.search(pattern, prediction)
+        if match:
+            return match.group(1).strip()
+        return prediction.strip()