evalscope 0.10.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

evalscope/benchmarks/tool_bench/utils.py

@@ -0,0 +1,203 @@
+import json
+
+from evalscope.metrics import compute_rouge_score_one_sample
+
+
+def evaluate_rougel(cand_list: list, ref_list: list):
+    if len(ref_list) == 0:
+        return 0
+    rouge_score = compute_rouge_score_one_sample(cand_list, ref_list)
+    rougel = rouge_score.get('rouge-l-f', 0)
+
+    return rougel
+
+
+def evaluate_action_em(cand_list: list, ref_list: list):
+    if len(ref_list) == 0:
+        return 0
+    em = 0
+    for cand, ref in zip(cand_list, ref_list):
+        em += (1 if cand == ref else 0)
+    return em / len(cand_list)
+
+
+def evaluate_action_input_f1(action_pred: list, action_ref: list, cand_list: list, ref_list: list):
+    easy_f1 = []
+    hard_f1 = []
+    f1 = []
+    for i in range(len(action_pred)):
+        ref_action = action_ref[i]
+        pred_action = action_pred[i]
+
+        ref_input = ref_list[i]
+        cand_input = cand_list[i]
+
+        if ref_action != pred_action:
+            easy_f1.append(0)
+            hard_f1.append(0)
+            f1.append(0)
+        else:
+            try:
+                ref_input_json = json.loads(ref_input)
+                try:
+                    cand_input_json = json.loads(cand_input)
+                    half_match = 0
+                    full_match = 0
+                    if ref_input_json == {}:
+                        if cand_input_json == {}:
+                            easy_f1.append(1)
+                            f1.append(1)
+                        else:
+                            easy_f1.append(0)
+                            f1.append(0)
+                    else:
+                        for k, v in ref_input_json.items():
+                            if k in cand_input_json.keys():
+                                if cand_input_json[k] == v:
+                                    full_match += 1
+                                else:
+                                    half_match += 1
+
+                        recall = (0.5 * half_match + full_match) / (len(ref_input_json) + 1e-30)
+                        precision = (0.5 * half_match + full_match) / (len(cand_input_json) + 1e-30)
+                        hard_f1.append((2 * recall * precision) / (recall + precision))
+                        f1.append((2 * recall * precision) / (recall + precision))
+                except Exception:
+                    # cand_input = cand_input.replace("\n","").replace("\"","")
+                    # ref_input = cand_input.replace("\n","").replace("\"","")
+                    # rouge = Rouge()
+                    # rouge_score = rouge.get_scores(hyps=[cand_input], refs=[ref_input], avg=True)
+                    if ref_input_json == {}:
+                        easy_f1.append(0)
+                    else:
+                        hard_f1.append(0)
+                    # hard_f1.append(rouge_score["rouge-l"]["f"])
+                    # f1.append(rouge_score["rouge-l"]["f"])
+                    f1.append(0)
+            except Exception:
+                pass
+
+    # 检查列表是否为空，如果为空则返回0
+    easy_f1_avg = sum(easy_f1) / len(easy_f1) if easy_f1 else 0
+    hard_f1_avg = sum(hard_f1) / len(hard_f1) if hard_f1 else 0
+    f1_avg = sum(f1) / len(f1) if f1 else 0
+
+    return easy_f1_avg, hard_f1_avg, f1_avg
+
+
+def parse_action(text):
+    action = 'None'
+    action_input = '{}'
+    if 'Action Input:' in text:
+        input_idx = text.rindex('Action Input:')
+        action_input = text[input_idx + len('Action Input:'):].strip()
+    else:
+        action_input = '{}'
+
+    if 'Action:' in text:
+        action_idx = text.rindex('Action:')
+        action = text[action_idx + len('Action:'):].strip()
+        if 'Action Input:' in action:
+            input_idx = action.index('Action Input:')
+            action = action[:input_idx].strip()
+    else:
+        action = 'none'
+    return action, action_input
+
+
+def parse_output(text):
+    action, action_input = parse_action(text)
+    if action == 'Finish':
+        try:
+            action_input = json.loads(action_input)
+            # print(action_input)
+            # print(json.dumps(action_input,indent=2))
+            return_type = action_input['return_type']
+            if return_type == 'give_answer':
+                if 'final_answer' in action_input.keys():
+                    answer = str(action_input['final_answer'])
+                    if answer.strip() in ['', '.', ',']:
+                        answer = 'None'
+                else:
+                    answer = 'None'
+                return 'finish', action, action_input, answer
+            else:
+                return 'give up', None, None, None
+        except Exception:
+            return 'give up', None, None, None
+    else:
+        plan = 'call'
+        answer = None
+        return plan, action, action_input, answer
+
+
+def calculate_metrics(data):
+    """
+    Calculate the metrics for the given data.
+    """
+    plan_ref = []
+    plan_pred = []
+    hallu_cases = []
+    answer_ref = []
+    action_ref = []
+    action_input_ref = []
+    answer_pred = []
+    action_pred = []
+    action_input_pred = []
+    hallu_pred = 0
+
+    reference = data['target']
+    prediction = data['predictions']
+    ref_plan, ref_action, ref_input, ref_ans = parse_output(reference)
+    # ref_plan: call
+    # ref_action: spott
+    # ref_input: {"is_id": "city center" }
+    # ref_ans: None
+
+    pred_plan, pred_action, pred_input, pred_ans = parse_output(prediction)
+    if ref_action is not None and ref_action == 'invalid_hallucination_function_name':
+        return {}
+    if pred_action is not None and ref_action != 'none' and ref_action not in [t['name'] for t in data['tools']]:
+        return {}
+
+    if pred_action is not None and pred_action != 'none' and pred_action not in [t['name'] for t in data['tools']]:
+        hallu_pred += 1
+        hallu_cases.append(data)
+
+    plan_ref.append(ref_plan)
+    plan_pred.append(pred_plan)
+    if ref_plan == 'give up':
+        pass
+    elif ref_plan == 'finish':
+        answer_ref.append(ref_ans)
+        if pred_ans is None:
+            answer_pred.append('none')
+        else:
+            answer_pred.append(pred_ans)
+    else:
+        action_ref.append(ref_action)
+        action_input_ref.append(ref_input)
+        if pred_action is None:
+            action_pred.append('none')
+        else:
+            action_pred.append(pred_action)
+
+        if pred_input is None:
+            action_input_pred.append('{}')
+        else:
+            action_input_pred.append(pred_input)
+
+    metric = {}
+    rouge = evaluate_rougel(answer_pred, answer_ref)
+    plan_em = evaluate_action_em(cand_list=plan_pred, ref_list=plan_ref)
+    action_em = evaluate_action_em(cand_list=action_pred, ref_list=action_ref)
+    easy_f1, hard_f1, f1 = evaluate_action_input_f1(action_pred, action_ref, action_input_pred, action_input_ref)
+    hallu_rate = hallu_pred
+    metric['Act.EM'] = action_em
+    metric['F1'] = f1
+    metric['HalluRate'] = hallu_rate
+    metric['plan_em'] = plan_em
+    metric['Easy_F1'] = easy_f1
+    metric['Hard_F1'] = hard_f1
+    metric['Rouge-L'] = rouge
+    return metric

evalscope/benchmarks/trivia_qa/trivia_qa_adapter.py

@@ -1,136 +1,74 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 # Copyright (c) EleutherAI Inc, and its affiliates.
-import csv
-import os
 
-from evalscope.benchmarks import Benchmark
-from evalscope.benchmarks.data_adapter import DataAdapter
-from evalscope.constants import EvalType
-from evalscope.metrics import AverageAccuracy
-from evalscope.models import ChatGenerationModelAdapter
-from evalscope.utils import get_logger
+from typing import Any, Dict
 
-# flake8: noqa
+from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.evaluator import TaskState
+from evalscope.api.metric import Score
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.logger import get_logger
 
 logger = get_logger()
 
+PROMPT_TEMPLATE = """
+Read the content and answer the following question.
 
-@Benchmark.register(
-    name='trivia_qa',
-    dataset_id='modelscope/trivia_qa',
-    model_adapter=ChatGenerationModelAdapter,
-    subset_list=['default'],
-    metric_list=[AverageAccuracy],
-    few_shot_num=5,
-    train_split='dev',
-    eval_split='test',
-)
-class TriviaQaAdapter(DataAdapter):
-
-    def __init__(self, **kwargs):
-
-        super().__init__(**kwargs)
+Content: {content}
 
-    def load_from_disk(self, dataset_name_or_path, subset_list, work_dir, **kwargs) -> dict:
-        data_dict = {}
-        for subset_name in subset_list:
-            data_dict[subset_name] = {}
-            for split in [self.train_split, self.eval_split]:
-                if os.path.exists(dataset_name_or_path):
-                    file_path = os.path.join(dataset_name_or_path, f'trivia-{split}.qa.csv')
-                else:
-                    file_path = os.path.join(work_dir, dataset_name_or_path, f'trivia-{split}.qa.csv')
-                if os.path.exists(file_path):
-                    with open(file_path, 'r', encoding='utf-8') as f:
-                        reader = csv.reader(f, delimiter='\t')
-                        split_data = []
-                        for row in reader:
-                            assert len(row) == 2
-                            question = row[0]
-                            answers = eval(row[1])
-                            split_data.append({
-                                'input': [{
-                                    'role': 'system',
-                                    'content': 'Follow the given examples and answer the question.'
-                                }, {
-                                    'role': 'user',
-                                    'content': question
-                                }],
-                                'ideal':
-                                answers
-                            })
-                        data_dict[subset_name][split] = split_data
+Question: {question}
 
-        return data_dict
+Keep your The last line of your response should be of the form "ANSWER: $ANSWER" (without quotes) where $ANSWER is the answer to the problem.
+""".lstrip()  # noqa: E501
 
-    def gen_prompt(self, input_d: dict, subset_name: str, few_shot_list: list, **kwargs) -> dict:
-        """
-        Generate model prompt from raw input, unify the prompt format for TriviaQA benchmark.
 
-        Args:
-            input_d (dict): The raw input. A single data format of the TriviaQA:
-
-            {
-                "input": [
-                    {"role": "system", "content": "Follow the given examples and answer the question."},
-                    {"role": "user", "content": "Which Lloyd Webber musical premiered in the US on 10th December 1993?"}
-                ],
-                "ideal": [
-                    "Sunset Blvd",
-                    "West Sunset Boulevard",
-                    "Sunset Boulevard",
-                    "Sunset Bulevard",
-                    "Sunset Blvd.",
-                    "sunset boulevard",
-                    "sunset bulevard",
-                    "west sunset boulevard",
-                    "sunset blvd"
-                ]
+@register_benchmark(
+    BenchmarkMeta(
+        name='trivia_qa',
+        pretty_name='TriviaQA',
+        dataset_id='evalscope/trivia_qa',
+        tags=[Tags.QA, Tags.READING_COMPREHENSION],
+        description=
+        'TriviaQA is a large-scale reading comprehension dataset consisting of question-answer pairs collected from trivia websites. It includes questions with multiple possible answers, making it suitable for evaluating the ability of models to understand and generate answers based on context.',  # noqa: E501
+        subset_list=['rc.wikipedia'],
+        few_shot_num=0,
+        train_split=None,
+        eval_split='validation',
+        metric_list=[{
+            'acc': {
+                'allow_inclusion': True
             }
+        }],
+        prompt_template=PROMPT_TEMPLATE,
+    )
+)
+class TriviaQaAdapter(DefaultDataAdapter):
 
-        Returns:
-            {'data': [(context, continuation), ...]}
-        """
-
-        def get_sys_prompt(inp: dict) -> str:
-            return inp['input'][0]['content']
-
-        prompt = get_sys_prompt(input_d)
-        few_shot_prompts = [self._generate_prompt(input_d=sample, include_answer=True) for sample in few_shot_list]
-        context: str = '\n'.join(few_shot_prompts) + '\n'
-        context += self._generate_prompt(input_d=input_d, include_answer=False)
-        full_prompt = context
-
-        return {'data': [full_prompt], 'system_prompt': prompt or self.prompt_template}
-
-    def get_gold_answer(self, input_d: dict) -> list:
-        # Get the gold choice
-        ans: list = input_d.get('ideal', [])
-        return ans
-
-    def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = EvalType.CHECKPOINT) -> str:
-        """
-        Parse the model output to get the answer.
-
-        Args:
-            result: Predicted answer from the model. A list of loglikelihood values for inputs pairs.
-            raw_input_d: The raw input. A single data format of the TriviaQA:
-            eval_type: The type of evaluation, e.g. 'checkpoint' or 'service' or 'custom'.
-
-        Returns:
-            The predicted answer.
-        """
-        return result
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
 
-    def match(self, gold: list, pred: str) -> float:
-        is_correct = any([cand in pred for cand in gold])
-        return 1 if is_correct else 0
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        question = record['question']
+        answers = record['answer']['aliases'] + record['answer']['normalized_aliases']
+        content = record['entity_pages']['wiki_context']
+        return Sample(
+            input=question, target=answers, metadata={
+                'question_id': record['question_id'],
+                'content': content
+            }
+        )
 
-    @classmethod
-    def _generate_prompt(cls, input_d: dict, include_answer=True) -> str:
+    def format_prompt_template(self, sample):
+        return self.prompt_template.format(content=sample.metadata['content'], question=sample.input)
 
-        example: str = f"Question: {input_d['input'][1]['content']}\nAnswer:"
-        if include_answer:
-            example += f" {input_d['ideal'][0]}\n\n"
+    def extract_answer(self, prediction: str, task_state: TaskState):
+        # use regex to extract the answer from the prediction
+        import re
 
-        return example
+        pattern = r'ANSWER:\s*(.*)'
+        match = re.search(pattern, prediction)
+        if match:
+            return match.group(1).strip()
+        return prediction.strip()