evalscope 0.10.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

evalscope/benchmarks/gsm8k/gsm8k_adapter.py

@@ -1,159 +1,92 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
-# Copyright (c) EleutherAI, Inc. and its affiliates.
-# flake8: noqa
-import math
-import os
+
 import re
+from typing import Any, Dict
 
-from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.metrics import AverageAccuracy
-from evalscope.models import ChatGenerationModelAdapter
-from evalscope.utils.io_utils import jsonl_to_list
+from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.evaluator import TaskState
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
 from evalscope.utils.logger import get_logger
 
 logger = get_logger()
 
+PROMPT_TEMPLATE = """
+Solve the following math problem step by step. The last line of your response should display the answer enclosed within \\boxed{{\\text{{$ANSWER}}}}.
+
+Example:
+
+Let's solve the problem step by step.
+
+Problem: Eliza's rate per hour for the first 40 hours she works each week is $10. She also receives an overtime pay of 1.2 times her regular hourly rate. If Eliza worked for 45 hours this week, how much are her earnings for this week?
+
+Step 1: Calculate Eliza's earnings for the first 40 hours. Eliza's hourly rate is $10, so her earnings for the first 40 hours are $10/hour x 40 hours = $400.
+Step 2: Calculate Eliza's overtime pay rate. Eliza's overtime pay rate is 1.2 times her regular hourly rate, so her overtime pay rate is $10/hour x 1.2 = $12/hour.
+Step 3: Calculate Eliza's earnings for the overtime hours. Eliza worked for 45 hours, so her overtime hours are 45 hours - 40 hours = 5 hours. Her earnings for the overtime hours are $12/hour x 5 hours = $60.
+Step 4: Calculate Eliza's total earnings for the week. Eliza's total earnings for the week are her earnings for the first 40 hours plus her earnings for the overtime hours, which is $400 + $60 = $460.
+
+Answer:
+\\boxed{{\\text{{460}}}}
+
+question:
+{question}
+
+Remember to put your answer on its own line at the end in the form "\\boxed{{\\text{{$ANSWER}}}}" (without quotes), where $ANSWER is replaced by the actual answer to the problem.
+""".lstrip()  # noqa: E501
+
+FEWSHOT_TEMPLATE = """
+Here are some examples of how to solve similar problems:
+
+{fewshot}
 
-@Benchmark.register(
-    name='gsm8k',
-    dataset_id='modelscope/gsm8k',
-    model_adapter=ChatGenerationModelAdapter,
-    subset_list=['main'],
-    metric_list=[AverageAccuracy],
-    few_shot_num=4,
-    train_split='train',
-    eval_split='test',
-    prompt_template='',
+""".lstrip() + PROMPT_TEMPLATE
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='gsm8k',
+        pretty_name='GSM8K',
+        dataset_id='AI-ModelScope/gsm8k',
+        tags=[Tags.MATH, Tags.REASONING],
+        description=
+        'GSM8K (Grade School Math 8K) is a dataset of grade school math problems, designed to evaluate the mathematical reasoning abilities of AI models.',  # noqa: E501
+        subset_list=['main'],
+        few_shot_num=4,
+        train_split='train',
+        eval_split='test',
+        metric_list=[{
+            'acc': {
+                'numeric': True
+            }
+        }],
+        prompt_template=PROMPT_TEMPLATE,
+        few_shot_prompt_template=FEWSHOT_TEMPLATE,
+    )
 )
-class GSM8KAdapter(DataAdapter):
+class GSM8KAdapter(DefaultDataAdapter):
 
     def __init__(self, **kwargs):
-        """
-        Data adapter for GSM8K dataset.
-
-        Args:
-            subset_list (list): Subset list for the dataset. Default: ['main']
-            metric_list (list): Metric list for the dataset. Default: [{'name': 'AverageAccuracy', 'object': mean}]
-            few_shot_num (int): Number of few-shot examples. Default: 4
-            train_split (str): Train split name. Default: 'train'
-            eval_split (str): The target eval split name. Default: 'test'
-            **kwargs: ...
-        """
-        few_shot_num = kwargs.get('few_shot_num', 4)
-        if few_shot_num != 4 and few_shot_num != 0:
-            logger.error(f'GSM8K uses 4-shot examples with CoT or 0-shot by system, but got {few_shot_num}. '
-                         f'Use 4-shot by default.')
-            kwargs['few_shot_num'] = 4
-
         super().__init__(**kwargs)
 
-    def load_from_disk(self, dataset_name_or_path, subset_list, work_dir, **kwargs) -> dict:
-        data_dict = {}
-        for subset_name in subset_list:
-            data_dict[subset_name] = {}
-            for split in [self.train_split, self.eval_split]:
-                if os.path.exists(dataset_name_or_path):
-                    file_path = os.path.join(dataset_name_or_path, f'{split}.jsonl')
-                else:
-                    file_path = os.path.join(work_dir, dataset_name_or_path, f'{split}.jsonl')
-                if os.path.exists(file_path):
-                    data_dict[subset_name][split] = jsonl_to_list(file_path)
-
-        return data_dict
-
-    def gen_prompt(self, input_d: dict, few_shot_list: list, **kwargs) -> dict:
-        """
-        Generate prompt for the model.
-
-        Args:
-            input_d (dict): The raw input. A single data format of the GSM8K:
-            {
-                "question": "Janet\\u2019s ducks lay 16 eggs per day. She eats three for breakfast every morning and bakes muffins for her friends every day with four. She sells the remainder at the farmers\' market daily for $2 per fresh duck egg. How much in dollars does she make every day at the farmers\' market?",
-                "answer": "Janet sells 16 - 3 - 4 = <<16-3-4=9>>9 duck eggs a day.\\nShe makes 9 * 2 = $<<9*2=18>>18 every day at the farmer\\u2019s market.\\n#### 18"
-            }
-        """
-        use_fewshot = self.few_shot_num > 0
-
-        full_prompt = self._generate_prompt(input_d, few_shot_list=few_shot_list, use_fewshot=use_fewshot)
-
-        return {'data': [full_prompt], 'system_prompt': self.prompt_template}
-
-    def get_gold_answer(self, input_d: dict) -> str:
-        # Extract the gold answer from the input dict.
-        ans: str = input_d.get('answer', '')
-        ans = self.extract_answer(ans).strip()
-        if not ans:
-            logger.error(f'No ground truth answer found in the input: {input_d}')
-        return ans
-
-    def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = 'checkpoint') -> str:
-        """
-        Parse the model output to get the answer. Could be the best choice index.
-
-        Args:
-            result: Predicted answer from the model. Usually a string for chat.
-            raw_input_d (dict): The raw input. Depending on the dataset.
-            eval_type (str): 'checkpoint' or 'service'
-
-        Returns:
-            The parsed answer. Depending on the dataset. Usually a string for chat.
-        """
-        # Note: to use same extraction method for both of checkpoint and custom.
-        return self.extract_answer(result)
-
-    def match(self, gold: str, pred: str) -> float:
-        """
-        Match the gold answer and predicted answer.
-
-        Args:
-            gold (str): The golden answer. Note: to be extracted.
-            pred (str): The extracted prediction. Usually a string for chat/multiple-choice-questions.
-                        e.g. 'B'
-        """
-
-        def number_equal(gold_ans, pred_ans):
-            if pred_ans is None:
-                return False
-            try:
-                return math.isclose(eval(gold_ans), eval(pred_ans), rel_tol=0, abs_tol=1e-4)
-            except:
-                logger.warning(f'##report##Cannot compare two numbers: gold_ans={gold_ans}, pred_ans={pred_ans}')
-                return False
-
-        return number_equal(gold_ans=gold, pred_ans=pred)
-
-    @classmethod
-    def _generate_prompt(cls, input_d: dict, few_shot_list: list, use_fewshot: bool = True) -> str:
-        if use_fewshot:
-            # Use 4-shot examples by system
-            context = (
-                "Question: Angelo and Melanie want to plan how many hours over the next week they should study together for their test next week. They have 2 chapters of their textbook to study and 4 worksheets to memorize. They figure out that they should dedicate 3 hours to each chapter of their textbook and 1.5 hours for each worksheet. If they plan to study no more than 4 hours each day, how many days should they plan to study total over the next week if they take a 10-minute break every hour, include 3 10-minute snack breaks each day, and 30 minutes for lunch each day?\nLet's think step by step\n"
-                'Angelo and Melanie think they should dedicate 3 hours to each of the 2 chapters, 3 hours x 2 chapters = 6 hours total.\nFor the worksheets they plan to dedicate 1.5 hours for each worksheet, 1.5 hours x 4 worksheets = 6 hours total.\nAngelo and Melanie need to start with planning 12 hours to study, at 4 hours a day, 12 / 4 = 3 days.\nHowever, they need to include time for breaks and lunch. Every hour they want to include a 10-minute break, so 12 total hours x 10 minutes = 120 extra minutes for breaks.\nThey also want to include 3 10-minute snack breaks, 3 x 10 minutes = 30 minutes.\nAnd they want to include 30 minutes for lunch each day, so 120 minutes for breaks + 30 minutes for snack breaks + 30 minutes for lunch = 180 minutes, or 180 / 60 minutes per hour = 3 extra hours.\nSo Angelo and Melanie want to plan 12 hours to study + 3 hours of breaks = 15 hours total.\nThey want to study no more than 4 hours each day, 15 hours / 4 hours each day = 3.75\nThey will need to plan to study 4 days to allow for all the time they need.\nThe answer is 4\n\n'
-                "Question: Mark's basketball team scores 25 2 pointers, 8 3 pointers and 10 free throws.  Their opponents score double the 2 pointers but half the 3 pointers and free throws.  What's the total number of points scored by both teams added together?\nLet's think step by step\n"
-                "Mark's team scores 25 2 pointers, meaning they scored 25*2= 50 points in 2 pointers.\nHis team also scores 6 3 pointers, meaning they scored 8*3= 24 points in 3 pointers\nThey scored 10 free throws, and free throws count as one point so they scored 10*1=10 points in free throws.\nAll together his team scored 50+24+10= 84 points\nMark's opponents scored double his team's number of 2 pointers, meaning they scored 50*2=100 points in 2 pointers.\nHis opponents scored half his team's number of 3 pointers, meaning they scored 24/2= 12 points in 3 pointers.\nThey also scored half Mark's team's points in free throws, meaning they scored 10/2=5 points in free throws.\nAll together Mark's opponents scored 100+12+5=117 points\nThe total score for the game is both team's scores added together, so it is 84+117=201 points\nThe answer is 201\n\n"
-                "Question: Bella has two times as many marbles as frisbees. She also has 20 more frisbees than deck cards. If she buys 2/5 times more of each item, what would be the total number of the items she will have if she currently has 60 marbles?\nLet's think step by step\n"
-                "When Bella buys 2/5 times more marbles, she'll have increased the number of marbles by 2/5*60 = 24\nThe total number of marbles she'll have is 60+24 = 84\nIf Bella currently has 60 marbles, and she has two times as many marbles as frisbees, she has 60/2 = 30 frisbees.\nIf Bella buys 2/5 times more frisbees, she'll have 2/5*30 = 12 more frisbees.\nThe total number of frisbees she'll have will increase to 30+12 = 42\nBella also has 20 more frisbees than deck cards, meaning she has 30-20 = 10 deck cards\nIf she buys 2/5 times more deck cards, she'll have 2/5*10 = 4 more deck cards.\nThe total number of deck cards she'll have is 10+4 = 14\nTogether, Bella will have a total of 14+42+84 = 140 items\nThe answer is 140\n\n"
-                "Question: A group of 4 fruit baskets contains 9 apples, 15 oranges, and 14 bananas in the first three baskets and 2 less of each fruit in the fourth basket. How many fruits are there?\nLet's think step by step\n"
-                'For the first three baskets, the number of apples and oranges in one basket is 9+15=24\nIn total, together with bananas, the number of fruits in one basket is 24+14=38 for the first three baskets.\nSince there are three baskets each having 38 fruits, there are 3*38=114 fruits in the first three baskets.\nThe number of apples in the fourth basket is 9-2=7\nThere are also 15-2=13 oranges in the fourth basket\nThe combined number of oranges and apples in the fourth basket is 13+7=20\nThe fourth basket also contains 14-2=12 bananas.\nIn total, the fourth basket has 20+12=32 fruits.\nThe four baskets together have 32+114=146 fruits.\nThe answer is 146\n\n'
-                f"Question: {input_d['question']}\nLet's think step by step\nAnswer:")
-            # context = input_d['question']
-            # fewshot_prompts = ['Question: ' + item_d['question'] + '\nAnswer: ' + item_d['answer'] for item_d in few_shot_list]
-            # fewshot_prompts = fewshot_prompts + ['Question: ' + context + '\nAnswer:']
-            # context = '\n\n'.join(fewshot_prompts)
-        else:
-            context = input_d['question']
-            context = 'Question: ' + context + '\nAnswer:'
-        return context
-
-    @staticmethod
-    def extract_answer(s: str) -> str:
-        _PAT_LAST_DIGIT = re.compile(r'([+-])?(?=([0-9]|\.[0-9]))(0|([1-9](\d{0,2}(,\d{3})*)|\d*))?(\.\d*)?(?=\D|$)')
-        match = list(_PAT_LAST_DIGIT.finditer(s))
-        if match:
-            last_digit = match[-1].group().replace(',', '').replace('+', '').strip().strip('.')
-            # print(f"The last digit in {s} is {last_digit}")
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        DELIM = '####'
+        question = record['question']
+        answer = record['answer'].split(DELIM)
+        target = answer.pop().strip()
+        reasoning = DELIM.join(answer)
+
+        return Sample(input=question, target=target, metadata={'reasoning': reasoning.strip()})
+
+    def sample_to_fewshot(self, sample: Sample) -> str:
+        if sample.metadata:
+            return (
+                f'{sample.input}\n\nReasoning:\n' + f"{sample.metadata['reasoning']}\n\n" + f'ANSWER: {sample.target}'
+            )
         else:
-            last_digit = None
-            print(f'No digits found in {s!r}', flush=True)
+            return ''
+
+    def extract_answer(self, prediction: str, task_state: TaskState):
+        from evalscope.metrics.math_parser import extract_answer
 
-        return last_digit
+        return extract_answer(prediction)

evalscope/benchmarks/hallusion_bench/hallusion_bench_adapter.py

@@ -0,0 +1,159 @@
+from collections import defaultdict
+from typing import Any, Dict, List
+
+from evalscope.api.benchmark import BenchmarkMeta, VisionLanguageAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.evaluator.state import TaskState
+from evalscope.api.messages import ChatMessageUser, Content, ContentImage, ContentText
+from evalscope.api.metric.scorer import AggScore, SampleScore, Score
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.io_utils import bytes_to_base64
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='hallusion_bench',
+        pretty_name='HallusionBench',
+        tags=[Tags.MULTI_MODAL, Tags.HALLUCINATION, Tags.YES_NO],
+        description=
+        'HallusionBench is an advanced diagnostic benchmark designed to evaluate image-context reasoning, analyze models\' tendencies for language hallucination and visual illusion in large vision-language models (LVLMs).',  # noqa: E501
+        dataset_id='lmms-lab/HallusionBench',
+        metric_list=['aAcc', 'qAcc', 'fAcc'],
+        aggregation='f1',
+        eval_split='image',
+        prompt_template='{question}\nPlease answer YES or NO without an explanation.',
+    )
+)
+class HallusionBenchAdapter(VisionLanguageAdapter):
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+
+        input_text = self.prompt_template.format(question=record['question'])
+        content_list: List[Content] = [ContentText(text=input_text)]
+        image = record.get('image')
+        if image:
+            image_base64 = bytes_to_base64(image['bytes'], format='png', add_header=True)
+            content_list.append(ContentImage(image=image_base64))
+        answer = 'NO' if str(record.get('answer', '0')) == '1' else 'YES'
+        return Sample(
+            input=[ChatMessageUser(content=content_list)],
+            target=answer,
+            metadata={
+                'category': record.get('category'),
+                'subcategory': record.get('subcategory'),
+                'visual_input': record.get('visual_input'),
+                'set_id': record.get('set_id'),
+                'figure_id': record.get('figure_id'),
+                'question_id': record.get('question_id'),
+            }
+        )
+
+    def match_score(self, original_prediction, filtered_prediction, reference, task_state) -> Score:
+        score = Score(
+            extracted_prediction=filtered_prediction,
+            prediction=original_prediction,
+        )
+        # Check if the reference answer is in the filtered prediction
+        result = 1 if reference in filtered_prediction.strip().upper() else 0
+        score.value = {'acc': result}
+        return score
+
+    def aggregate_scores(self, sample_scores: List[SampleScore]) -> List[AggScore]:
+
+        def compute_aAcc(scores: List[SampleScore]):
+            total = len(scores)
+            if total == 0:
+                return 0.0, 0
+            correct = sum(ss.score.main_value for ss in scores)
+            return (correct / total), total
+
+        def compute_group_accuracy(scores: List[SampleScore], group_type: str):
+            # group_type: 'figure' or 'question'
+            groups = defaultdict(list)
+            for ss in scores:
+                md = ss.sample_metadata
+                subcategory = md.get('subcategory')
+                set_id = md.get('set_id')
+                group_id = md.get('figure_id') if group_type == 'figure' else md.get('question_id')
+                if subcategory is None or set_id is None or group_id is None:
+                    # Skip incomplete records for this grouping
+                    continue
+                key = f'{subcategory}_{set_id}_{group_id}'
+                groups[key].append(ss.score.main_value)
+            if not groups:
+                return 0.0, 0
+            num_correct_groups = sum(1 for vals in groups.values() if all(vals))
+            num_groups = len(groups)
+            return (num_correct_groups / num_groups), num_groups
+
+        def compute_metrics(scores: List[SampleScore]) -> Dict[str, Dict[str, float]]:
+            a_acc, a_n = compute_aAcc(scores)
+            f_acc, f_n = compute_group_accuracy(scores, 'figure')
+            q_acc, q_n = compute_group_accuracy(scores, 'question')
+            return {
+                'aAcc': {
+                    'score': a_acc,
+                    'num': a_n
+                },
+                'fAcc': {
+                    'score': f_acc,
+                    'num': f_n
+                },
+                'qAcc': {
+                    'score': q_acc,
+                    'num': q_n
+                },
+            }
+
+        outputs: List[AggScore] = []
+
+        # By subcategory
+        subcategories = sorted({ss.sample_metadata.get('subcategory') for ss in sample_scores})
+        for subcategory in subcategories:
+            subset = [ss for ss in sample_scores if ss.sample_metadata.get('subcategory') == subcategory]
+            stats = compute_metrics(subset)
+            for metric in ['aAcc', 'fAcc', 'qAcc']:
+                outputs.append(
+                    AggScore(
+                        score=stats[metric]['score'],
+                        metric_name=metric,
+                        aggregation_name=str(subcategory),
+                        num=stats[metric]['num'],
+                    )
+                )
+
+        # By category
+        categories = sorted({ss.sample_metadata.get('category') for ss in sample_scores})
+        for category in categories:
+            subset = [ss for ss in sample_scores if ss.sample_metadata.get('category') == category]
+            stats = compute_metrics(subset)
+            for metric in ['aAcc', 'fAcc', 'qAcc']:
+                outputs.append(
+                    AggScore(
+                        score=stats[metric]['score'],
+                        metric_name=metric,
+                        aggregation_name=str(category),
+                        num=stats[metric]['num'],
+                    )
+                )
+
+        # Overall
+        overall = compute_metrics(sample_scores)
+        for metric in ['aAcc', 'fAcc', 'qAcc']:
+            outputs.append(
+                AggScore(
+                    score=overall[metric]['score'],
+                    metric_name=metric,
+                    aggregation_name='Overall',
+                    num=overall[metric]['num'],
+                )
+            )
+
+        return outputs

evalscope/benchmarks/halu_eval/halu_eval_adapter.py

@@ -0,0 +1,128 @@
+# flake8: noqa: E501
+
+from typing import Any, Dict, List
+
+from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.messages import ChatMessageUser, Content, ContentText
+from evalscope.api.metric.scorer import AggScore, SampleScore, Score
+from evalscope.api.registry import register_benchmark
+from evalscope.benchmarks.halu_eval.halu_eval_instructions import (
+    DIALOGUE_INSTRUCTIONS,
+    QA_INSTRUCTIONS,
+    SUMMARIZATION_INSTRUCTIONS,
+)
+from evalscope.constants import Tags
+from evalscope.utils.logger import get_logger
+
+DESCRIPTION = (
+    'HaluEval is a large collection of generated and human-annotated hallucinated samples for evaluating the performance of LLMs in recognizing hallucination.'
+)
+
+logger = get_logger()
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='halueval',
+        pretty_name='HaluEval',
+        tags=[Tags.KNOWLEDGE, Tags.HALLUCINATION, Tags.YES_NO],
+        description=DESCRIPTION.strip(),
+        dataset_id='evalscope/HaluEval',
+        subset_list=['dialogue_samples', 'qa_samples', 'summarization_samples'],
+        default_subset='Full',
+        metric_list=['accuracy', 'precision', 'recall', 'f1_score', 'yes_ratio'],
+        few_shot_num=0,
+        eval_split='data',
+        prompt_template='{question}'
+    )
+)
+class HaluEvalAdapter(DefaultDataAdapter):
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.add_overall_metric = False
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        if self.current_subset_name == 'dialogue_samples':
+            knowledge = record['knowledge']
+            dialogue_history = record['dialogue_history']
+            response = record['response']
+            hallucination = record['hallucination']
+            inputs = f'{DIALOGUE_INSTRUCTIONS}\n\n#Knowledge: {knowledge}\n#Dialogue History#: {dialogue_history}\n#Response#: {response}\n#Your Judgement#:'
+        elif self.current_subset_name == 'qa_samples':
+            knowledge = record['knowledge']
+            question = record['question']
+            answer = record['answer']
+            hallucination = record['hallucination']
+            inputs = f'{QA_INSTRUCTIONS}\n\n#Knowledge: {knowledge}\n#Question#: {question}\n#Answer#: {answer}\n#Your Judgement#:'
+        elif self.current_subset_name == 'summarization_samples':
+            document = record['document']
+            summary = record['summary']
+            hallucination = record['hallucination']
+            inputs = f'{SUMMARIZATION_INSTRUCTIONS}\n\n#Document#: {document}\n#Summary#: {summary}\n#Your Judgement#:'
+
+        input_text = self.prompt_template.format(question=inputs)
+        content_list: List[Content] = [ContentText(text=input_text)]
+        answer = str(hallucination).upper()  # 'YES' or 'NO'
+        return Sample(
+            input=[ChatMessageUser(content=content_list)], target=answer, metadata={
+                'answer': hallucination,
+            }
+        )
+
+    def match_score(self, original_prediction, filtered_prediction, reference, task_state) -> Score:
+        score = Score(
+            extracted_prediction=filtered_prediction,
+            prediction=original_prediction,
+        )
+        # Check if the reference answer is in the filtered prediction
+        result = 1 if reference in filtered_prediction.strip().upper() else 0
+        score.value = {'acc': result}
+        return score
+
+    def aggregate_scores(self, sample_scores: List[SampleScore]) -> List[AggScore]:
+        """
+        Custom aggregation to compute accuracy, precision, recall, f1_score, and yes_ratio.
+        """
+
+        def compute_metrics(scores: List[SampleScore]):
+            tp = fp = tn = fn = 0
+            yes_count = 0
+            total_count = len(scores)
+
+            for ss in scores:
+                gt = ss.sample_metadata['answer'].strip().upper()
+                # Get prediction based on score
+                pred = gt if ss.score.main_value == 1 else ('NO' if gt == 'YES' else 'YES')
+                if pred == 'YES':
+                    yes_count += 1
+                if pred == 'YES' and gt == 'YES':
+                    tp += 1
+                elif pred == 'YES' and gt == 'NO':
+                    fp += 1
+                elif pred == 'NO' and gt == 'NO':
+                    tn += 1
+                elif pred == 'NO' and gt == 'YES':
+                    fn += 1
+
+            accuracy = (tp + tn) / total_count if total_count > 0 else 0.0
+            precision = tp / (tp + fp) if (tp + fp) > 0 else 0.0
+            recall = tp / (tp + fn) if (tp + fn) > 0 else 0.0
+            f1_score = (2 * precision * recall) / (precision + recall) if (precision + recall) > 0 else 0.0
+            yes_ratio = yes_count / total_count if total_count > 0 else 0.0
+
+            return {
+                'accuracy': accuracy,
+                'precision': precision,
+                'recall': recall,
+                'f1_score': f1_score,
+                'yes_ratio': yes_ratio
+            }
+
+        overall_metrics = compute_metrics(sample_scores)
+        agg_scores = []
+        for metric_name, value in overall_metrics.items():
+            agg_scores.append(AggScore(metric_name=metric_name, score=value, num=len(sample_scores), metadata={}))
+
+        return agg_scores

evalscope/benchmarks/halu_eval/halu_eval_instructions.py

@@ -0,0 +1,84 @@
+# flake8: noqa: E501
+
+DIALOGUE_INSTRUCTIONS = """I want you act as a response judge. Given a dialogue history and a response, your objective is to determine if the provided response contains non-factual or hallucinated information. You SHOULD give your judgement based on the following hallucination types and the world knowledge.
+
+You are trying to determine if the true entity in the response is replaced with a highly similar entity.
+#Dialogue History#: [Human]: Could you recommand movies similar to The Dark Knight? [Assistant]: The sequel to Batman Begins is The Dark Knight. [Human]: Okay. Who is the director of The Dark Knight and any other movies from him not related to Batman?
+#Response#: Christopher Nolan was the director. He also directed insomnia and inception.
+#Your Judgement#: No
+#Dialogue History#: [Human]: Could you recommand movies similar to The Dark Knight? [Assistant]: The sequel to Batman Begins is The Dark Knight. [Human]: Okay. Who is the director of The Dark Knight and any other movies from him not related to Batman?
+#Response#: Steven Spielberg was the director. He also directed insomnia and inception.
+#Your Judgement#: Yes
+
+You are trying to determine if the true entity in the response is replaced with a dissimilar entity.
+#Dialogue History#: [Human]: Could you recommand movies similar to The Dark Knight? [Assistant]: The sequel to Batman Begins is The Dark Knight. [Human]: Okay. Who is the director of The Dark Knight and any other movies from him not related to Batman?
+#Response#: Christopher Nolan was the director. He also directed insomnia and inception.
+#Your Judgement#: No
+#Dialogue History#: [Human]: Could you recommand movies similar to The Dark Knight? [Assistant]: The sequel to Batman Begins is The Dark Knight. [Human]: Okay. Who is the director of The Dark Knight and any other movies from him not related to Batman?
+#Response#: Batman Begins was the director. He also directed insomnia and inception.
+#Your Judgement#: Yes
+
+You are trying to determine if the true entity in the response is replaced with a dissimilar entity in a different entity type.
+#Dialogue History#: [Human]: Could you recommand movies similar to The Dark Knight? [Assistant]: The sequel to Batman Begins is The Dark Knight. [Human]: Okay. Who is the director of The Dark Knight and any other movies from him not related to Batman?
+#Response#: Christopher Nolan was the director. He also directed insomnia and inception.
+#Your Judgement#: No
+#Dialogue History#: [Human]: Could you recommand movies similar to The Dark Knight? [Assistant]: The sequel to Batman Begins is The Dark Knight. [Human]: Okay. Who is the director of The Dark Knight and any other movies from him not related to Batman?
+#Response#: United States of America was the director. He also directed insomnia and inception.
+#Your Judgement#: Yes
+
+You should try your best to determine if the response contains non-factual or hallucinated information according to the above hallucination types. The answer you give MUST be \"Yes\" or \"No\"".
+"""
+
+QA_INSTRUCTIONS = """I want you act as an answer judge. Given a question and an answer, your objective is to determine if the provided answer contains non-factual or hallucinated information. You SHOULD give your judgement based on the following hallucination types and the world knowledge.
+
+You are trying to determine if the answer misunderstands the question context and intention.
+#Question#: What is a rare breed of dog that was derived as a variant of Rat Terrier, Shiloh Shepherd dog or American Hairless Terrier?
+#Answer#: American Hairless Terrier
+#Your Judgement#: No
+
+You are trying to determine if there is a factual contradiction between the answer and the world knowledge. Some information in the answer might be fabricated.
+#Question#: Are the New Orleans Outfall Canals the same length as the Augusta Canal?
+#Answer#: No, the New Orleans Outfall Canals and the Augusta Canal are not the same length. The Orleans Canal is approximately 3.6 miles (5.8 kilometers) long while the Augusta Canal is approximately 7 miles (11.3 kilometers) long.
+#Your Judgement#: Yes
+#Question#: What U.S Highway gives access to Zilpo Road, and is also known as Midland Trail?
+#Answer#: U.S Highway 70
+#Your Judgement#: Yes
+
+You are trying to determine if the answer is too general or too specific to answer the question at an appropriate level of specificity.
+#Question#: What genre do Superheaven and Oceansize belong to?
+#Answer#: Superheaven and Oceansize belong to the rock genre.
+#Your Judgement#: No
+#Question#: What profession do Kōbō Abe and Agatha Christie share?
+#Answer#: Playwright.
+#Your Judgement#: No
+
+You are trying to determine if the answer can be correctly inferred from the knowledge.
+#Question#: Which band has more members, Muse or The Raconteurs?
+#Answer#: Muse has more members than The Raconteurs.
+#Your Judgement#: Yes
+#Question#: Which is currently more valuable, Temagami-Lorrain Mine or Meadowbank Gold Mine?
+#Answer#: Meadowbank Gold Mine, since Meadowbank Gold Mine is still producing gold and the TemagamiLorrain Mine has been inactive for years.
+#Your Judgement#: No
+
+You should try your best to determine if the answer contains non-factual or hallucinated information according to the above hallucination types. The answer you give MUST be \"Yes\" or \"No\"".
+"""
+
+SUMMARIZATION_INSTRUCTIONS = """I want you act as a summary judge. Given a document and a summary, your objective is to determine if the provided summary contains non-factual or hallucinated information. You SHOULD give your judgement based on the following hallucination types and the world knowledge.
+
+You are trying to determine if the summary is factual but some information cannot be directly inferred or entailed from the document.
+#Document#: The panther chameleon was found on Monday by a dog walker in the wooded area at Marl Park. It had to be put down after X-rays showed all of its legs were broken and it had a deformed spine. RSPCA Cymru said it was an "extremely sad example of an abandoned and neglected exotic pet". Inspector Selina Chan said: "It is a possibility that the owners took on this animal but were unable to provide the care he needs and decided to release him to the wild. "We are urging potential owners of exotic animals to thoroughly research what is required in the care of the particular species before taking one on. "Potential owners need to make sure they can give their animal the environment it needs and they have the facilities, time, financial means and long-term commitment to maintain a good standard of care, as required under the Animal Welfare Act 2006." She added it was illegal to release non-native species into the wild.
+#Summary#: A chameleon that was found in a Cardiff park has been put down after being abandoned and neglected by its owners.
+#Your Judgement#: Yes
+
+You are trying to determine if there exists some non-factual and incorrect information in the summary.
+#Document#: The city was brought to a standstill on 15 December last year when a gunman held 18 hostages for 17 hours. Family members of victims Tori Johnson and Katrina Dawson were in attendance. Images of the floral tributes that filled the city centre in the wake of the siege were projected on to the cafe and surrounding buildings in an emotional twilight ceremony. Prime Minister Malcolm Turnbull gave an address saying a "whole nation resolved to answer hatred with love". "Testament to the spirit of Australians is that with such unnecessary, thoughtless tragedy, an amazing birth of mateship, unity and love occurs. Proud to be Australian," he said. How the Sydney siege unfolded New South Wales Premier Mike Baird has also announced plans for a permanent memorial to be built into the pavement in Martin Place. Clear cubes containing flowers will be embedded into the concrete and will shine with specialised lighting. It is a project inspired by the massive floral tributes that were left in the days after the siege. "Something remarkable happened here. As a city we were drawn to Martin Place. We came in shock and in sorrow but every step we took was with purpose," he said on Tuesday.
+#Summary#: Crowds have gathered in Sydney's Martin Place to honour the victims of the Lindt cafe siege, one year on.
+#Your Judgement#: No
+
+You are trying to determine if there is a factual contradiction between the summary and the document.
+#Document#: Christopher Huxtable, 34, from Swansea, had been missing since the collapse in February. His body was found on Wednesday and workers who carried out the search formed a guard of honour as it was driven from the site in the early hours of the morning. Ken Cresswell, 57, and John Shaw, 61, both from Rotherham, remain missing. The body of a fourth man, Michael Collings, 53, from Brotton, Teesside, was previously recovered from the site. Swansea East MP Carolyn Harris, who has been involved with the family since the incident, said they still did not know all the facts about the collapse. She said: "I feel very sad. My heart and my prayers go out to the family who have waited desperately for Christopher's body to be found. They can finally have closure, and say goodbye to him and grieve his loss. "But let's not forget that there's two other families who are still waiting for their loved ones to be returned." The building was due for demolition when it partially collapsed in February.
+#Summary#: The body of a man whose body was found at the site of the Swansea Bay Power Station collapse has been removed from the site.
+#Your Judgement#: Yes
+
+You should try your best to determine if the summary contains non-factual or hallucinated information according to the above hallucination types. The answer you give MUST be \"Yes\" or \"No\"".
+"""