evalscope 0.17.1__py3-none-any.whl → 1.0.0__py3-none-any.whl

evalscope/benchmarks/general_qa/general_qa_adapter.py

@@ -1,155 +1,94 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
-import os.path
-from collections import defaultdict
-from typing import List, Optional, Union
-
-from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.metrics import mean
-from evalscope.utils.io_utils import jsonl_to_list
+from typing import Any, Dict, List, Optional, Union
+
+from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.evaluator import TaskState
+from evalscope.api.messages import ChatMessageSystem, ChatMessageUser, dict_to_chat_message
+from evalscope.api.metric import Score
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
 from evalscope.utils.logger import get_logger
 
 logger = get_logger()
 
-
-@Benchmark.register(
-    name='general_qa',
-    pretty_name='General-QA',
-    description='A general question answering dataset for custom evaluation. '
-    'For detailed instructions on how to use this benchmark, please refer to the [User Guide](https://evalscope.readthedocs.io/zh-cn/latest/advanced_guides/custom_dataset/llm.html#qa).',  # noqa: E501
-    tags=['QA', 'Custom'],
-    dataset_id='general_qa',
-    subset_list=['default'],
-    metric_list=['AverageBLEU', 'AverageRouge'],
-    few_shot_num=0,
-    train_split=None,
-    eval_split='test',
-    prompt_template='请回答问题\n{query}',
+PROMPT_TEMPLATE = '请回答问题\n{question}'
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='general_qa',
+        pretty_name='General-QA',
+        description='A general question answering dataset for custom evaluation. '
+        'For detailed instructions on how to use this benchmark, please refer to the [User Guide](https://evalscope.readthedocs.io/zh-cn/latest/advanced_guides/custom_dataset/llm.html#qa).',  # noqa: E501
+        tags=[Tags.QA, Tags.CUSTOM],
+        dataset_id='general_qa',
+        metric_list=['BLEU', 'Rouge'],
+        few_shot_num=0,
+        train_split=None,
+        eval_split='test',
+        prompt_template=PROMPT_TEMPLATE,
+    )
 )
-class GeneralQAAdapter(DataAdapter):
+class GeneralQAAdapter(DefaultDataAdapter):
 
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
 
-    def load(self, dataset_name_or_path: str = None, subset_list: list = None, **kwargs) -> dict:
-        """
-        Load dataset from the given path or dataset name.
-
-        Args:
-            dataset_name_or_path (str): Path to dataset directory or file.
-            subset_list (list): List of subset names to load.
-
-        Returns:
-            dict: Loaded dataset organized by subset.
-        """
-        dataset_name_or_path = dataset_name_or_path or self.dataset_id
-        subset_list = subset_list or self.subset_list
-
-        data_file_dict = defaultdict(str)
-        data_item_dict = defaultdict(list)
-
-        # get data file path and subset name
-        if os.path.isdir(dataset_name_or_path):
-            for subset_name in subset_list:
-                data_file_dict[subset_name] = os.path.join(dataset_name_or_path, f'{subset_name}.jsonl')
-        elif os.path.isfile(dataset_name_or_path):
-            cur_subset_name = os.path.splitext(os.path.basename(dataset_name_or_path))[0]
-            data_file_dict[cur_subset_name] = dataset_name_or_path
-        else:
-            raise ValueError(f'Invalid dataset path: {dataset_name_or_path}')
-
-        # load data from local disk
-        try:
-            for subset_name, file_path in data_file_dict.items():
-                data_item_dict[subset_name] = jsonl_to_list(file_path)
-        except Exception as e:
-            raise ValueError(f'Failed to load data from {self.dataset_id}, got error: {e}')
-
-        data_dict = {subset_name: {'test': data_item_dict[subset_name]} for subset_name in data_file_dict.keys()}
-
-        return data_dict
-
-    def gen_prompt(self, input_d: dict, subset_name: str, few_shot_list: list, **kwargs) -> dict:
-        """
-        Generate prompt for the model based on input data.
-
-        Args:
-            input_d (dict): Input data dictionary.
-            subset_name (str): Name of the subset.
-            few_shot_list (list): List of few-shot examples.
-
-        Returns:
-            dict: Dictionary containing the generated prompt.
-        """
-        messages = input_d.get('messages')
-        query = input_d.get('question', '') or input_d.get('query', '')
-        system_prompt = input_d.get('system')
-        prompt = self.prompt_template.format(query=query)
-        return self.gen_prompt_data(prompt, system_prompt=system_prompt, messages=messages)
-
-    def get_gold_answer(self, input_d: dict) -> str:
-        """
-        Extract the gold (reference) answer from the input data.
-
-        Args:
-            input_d (dict): Input data dictionary.
-
-        Returns:
-            str: Gold answer string.
-        """
-        return input_d.get('answer') or input_d.get('response')
-
-    def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = 'checkpoint') -> str:
-        """
-        Parse the prediction result.
-
-        Args:
-            result (str): Model prediction result.
-            raw_input_d (dict, optional): Original input data.
-            eval_type (str): Evaluation type.
+    def load_from_disk(self, **kwargs):
+        return super().load_from_disk(use_local_loader=True)
 
-        Returns:
-            str: Parsed prediction result.
-        """
-        return result
-
-    def match(self, gold: str, pred: str) -> dict:
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
         """
-        Compute metric scores between gold and predicted answers.
+        Convert a data record to a Sample object.
 
         Args:
-            gold (str): Gold answer.
-            pred (str): Predicted answer.
+            record (Dict[str, Any]): Input data record.
 
         Returns:
-            dict: Dictionary of computed metric scores.
+            Sample: Sample object with input, target, and metadata.
         """
-        # reference free metrics
-        if gold is None:
-            return {'AverageAccuracy': -1}
-
-        # calculate rouge and bleu scores
-        res = dict()
-        if 'AverageRouge' in self.metric_list:
-            from evalscope.metrics.rouge_metric import compute_rouge_score_one_sample_zh
-
-            rouge_dict = compute_rouge_score_one_sample_zh([pred], [gold])
-            res.update(rouge_dict)
-        if 'AverageBLEU' in self.metric_list:
-            from evalscope.metrics import bleu_ngram_one_sample
+        query = record.get('question') or record.get('query')
+        answer = record.get('answer') or record.get('response')
+        system_prompt = record.get('system')
+        messages = record.get('messages')
+
+        message_list = []
+        if messages:
+            message_list = [dict_to_chat_message(m) for m in messages]
+        else:
+            if system_prompt:
+                message_list.append(ChatMessageSystem(content=system_prompt))
+            message_list.append(ChatMessageUser(content=query))
 
-            bleu_dict = bleu_ngram_one_sample(pred, gold)
-            res.update(bleu_dict)
-        return res
+        return Sample(input=message_list, target=answer or '')
 
-    def compute_metric(self, review_res_list: Union[List[dict], List[List[dict]]], **kwargs) -> List[dict]:
+    def match_score(
+        self, original_prediction: str, filtered_prediction: str, reference: str, task_state: TaskState
+    ) -> Score:
         """
-        Compute weighted mean of the metric scores for all samples.
-
-        Args:
-            review_res_list (list): List of metric score dictionaries.
-
-        Returns:
-            list: List of dictionaries with averaged metric results.
+        Calculate evaluation scores by comparing prediction with reference.
         """
-        items = super().compute_dict_metric(review_res_list, **kwargs)
-        return [{'metric_name': k, 'score': mean(v), 'num': len(v)} for k, v in items.items()]
+        # Initialize the score object with prediction details
+        score = Score(
+            extracted_prediction=filtered_prediction,
+            prediction=original_prediction,
+        )
+
+        # Calculate scores for each configured metric
+        for metric in self.metric_list:
+            try:
+                if metric == 'Rouge':
+                    from evalscope.metrics.rouge_metric import compute_rouge_score_one_sample_zh
+
+                    score.value.update(compute_rouge_score_one_sample_zh([filtered_prediction], [reference]))
+                elif metric == 'BLEU':
+                    from evalscope.metrics import bleu_ngram_one_sample
+
+                    score.value.update(bleu_ngram_one_sample(filtered_prediction, reference))
+            except Exception as e:
+                logger.error(f'Error calculating metric {metric}: {e}')
+                return None
+
+        score.main_score_name = 'Rouge-L-R'
+        return score

evalscope/benchmarks/gpqa/gpqa_adapter.py

@@ -1,63 +1,70 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
 import os
 import random
 import re
-
-from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.constants import EvalType, OutputType
-from evalscope.metrics import exact_match
-
-
-@Benchmark.register(
-    name='gpqa',
-    pretty_name='GPQA',
-    tags=['MCQ', 'Knowledge'],
-    description=
-    'GPQA is a dataset for evaluating the reasoning ability of large language models (LLMs) on complex mathematical problems. It contains questions that require step-by-step reasoning to arrive at the correct answer.',  # noqa: E501
-    dataset_id='modelscope/gpqa',
-    model_adapter=OutputType.GENERATION,
-    output_types=[OutputType.MULTIPLE_CHOICE, OutputType.GENERATION],
-    subset_list=['gpqa_extended', 'gpqa_main', 'gpqa_diamond'],
-    metric_list=['AveragePass@1'],
-    few_shot_num=5,
-    train_split=None,
-    eval_split='train',  # only have train split
-    prompt_template='{query}\nPlease reason step by step, and put your final answer within \\boxed{{}}.',
+from typing import Any, Dict
+
+from evalscope.api.benchmark import BenchmarkMeta, MultiChoiceAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.logger import get_logger
+from evalscope.utils.multi_choices import FEW_SHOT_TEMPLATE, MultipleChoiceTemplate
+
+logger = get_logger()
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='gpqa_diamond',
+        pretty_name='GPQA-Diamond',
+        tags=[Tags.KNOWLEDGE, Tags.MULTIPLE_CHOICE],
+        description=
+        'GPQA is a dataset for evaluating the reasoning ability of large language models (LLMs) on complex mathematical problems. It contains questions that require step-by-step reasoning to arrive at the correct answer.',  # noqa: E501
+        dataset_id='AI-ModelScope/gpqa_diamond',
+        metric_list=['acc'],
+        few_shot_num=0,
+        train_split=None,
+        eval_split='train',  # only have train split
+        prompt_template=MultipleChoiceTemplate.SINGLE_ANSWER_COT,
+    )
 )
-class GPQAAdapter(DataAdapter):
+class GPQAAdapter(MultiChoiceAdapter):
 
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
 
-        self.choices = ['A', 'B', 'C', 'D']
-        if self.few_shot_num and self.few_shot_num > 0:
-            self.prompt_prefix = 'Here are some example questions from experts. Answer the final question yourself, following the format of the previous questions exactly.\n'  # noqa: E501
-            self.prompt_prefix += open(
-                os.path.join(os.path.dirname(__file__), 'chain_of_thought.txt'), 'r',
-                encoding='utf-8').read() + '\nQuestion: '
-        else:
-            self.prompt_prefix = 'What is the correct answer to this question:'
-
-    def gen_prompt(self, input_d: dict, subset_name: str, few_shot_list: list, **kwargs) -> dict:
-        """
-        Generate model prompt from input data.
-        example:
-        {
-            "question":"Two people are playing the following game. A fair coin is tossed into the air. Person A says that in a single toss of the coin, the tail will come. So it's like the first shot or the third shot or the fifth shot. Person B says that the coin will come with a double toss. So like the second, fourth, sixth or eighth shot. Imagine this game played forever. What is the probability that person A wins this game?",
-            "choice1":"1/2",
-            "choice2":"1/4",
-            "choice3":"2/3",
-            "choice4":"1/8",
-            "answer":"C",
-        }
-        """  # noqa: E501
-        processed_input_d = self.__process_input(input_d)
-        input_d['answer'] = processed_input_d['answer']  # add answer to input_d for answer extraction
-        query = self.prompt_prefix + f"{input_d['Question']}\n{self.__form_options(processed_input_d['choices'])}"  # noqa: E501
-
-        prompt = self.prompt_template.format(query=query)
-        return self.gen_prompt_data(prompt)
-
-    def __process_input(self, input_d: dict) -> dict:
+        if self.few_shot_num > 0 and self.few_shot_num != 5:
+            logger.warning(
+                f'Only support few_shot_num 0 or 5 for {self.dataset_id}, but got {self.few_shot_num}. Use 5-shot by default.'  # noqa: E501
+            )
+            self.few_shot_num = 5
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        # Process the input to create shuffled choices and correct answer
+        processed_data = self._process_input(record)
+
+        return Sample(
+            input=record['Question'],
+            choices=processed_data['choices'],
+            target=processed_data['answer'],
+            subset_key=record.get('subset', ''),
+            metadata={
+                'correct_answer':
+                record['Correct Answer'],
+                'incorrect_answers':
+                [record['Incorrect Answer 1'], record['Incorrect Answer 2'], record['Incorrect Answer 3']],
+            },
+        )
+
+    def format_fewshot_template(self, fewshot, sample):
+        from .prompt import FEW_SHOT_SAMPLES
+
+        return FEW_SHOT_TEMPLATE.format(fewshot=FEW_SHOT_SAMPLES, ) + self.format_prompt_template(sample)
+
+    def _process_input(self, input_d: dict) -> dict:
+        """Process input to shuffle choices and determine correct answer letter."""
 
         def preprocess(text):
             if text is None:
@@ -77,53 +84,7 @@ class GPQAAdapter(DataAdapter):
         random.shuffle(choices)
         correct_answer_index = choices.index(preprocess(input_d['Correct Answer']))
 
-        out_doc = {
-            'choices': [choices[0], choices[1], choices[2], choices[3]],
+        return {
+            'choices': choices,
             'answer': f'{chr(65 + correct_answer_index)}',
         }
-        return out_doc
-
-    def __form_options(self, options: list):
-        option_str = 'Choices:\n'
-        for opt, choice in zip(options, self.choices):
-            option_str += f'({choice}) {opt}' + '\n'
-        return option_str
-
-    def get_gold_answer(self, input_d: dict) -> str:
-        """
-        Parse the raw input labels (gold).
-        """
-        return input_d['answer']
-
-    def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = EvalType.CHECKPOINT) -> str:
-        """
-        Parse the predicted result and extract proper answer.
-        """
-        if self.model_adapter == OutputType.MULTIPLE_CHOICE:
-            return result
-        else:
-            return GPQAAdapter.get_multiple_choice_answer(result)
-
-    def match(self, gold: str, pred: str) -> float:
-        """
-        Match the gold answer and the predicted answer.
-        """
-        return exact_match(gold=gold, pred=pred)
-
-    @staticmethod
-    def get_multiple_choice_answer(pred: str):
-        tmp = re.findall(r'\b(A|B|C|D)\b', pred.upper())
-        if tmp:
-            pred = tmp
-        else:
-            pred = [pred.strip().strip('.')]
-
-        if len(pred) == 0:
-            pred = ''
-        else:
-            pred = pred[-1]
-
-        # Remove the period at the end, again!
-        pred = pred.rstrip('.').rstrip('/')
-
-        return pred

evalscope/benchmarks/gpqa/{chain_of_thought.txt → prompt.py}

@@ -1,3 +1,5 @@
+# flake8: noqa
+FEW_SHOT_SAMPLES = """
 Question: In a given population, 1 out of every 400 people has a cancer caused by a completely recessive allele, b. Assuming the population is in Hardy-Weinberg equilibrium, which of the following is the expected proportion of individuals who carry the b allele but are not expected to develop the cancer?
 Choices:
 (A) 1/400
@@ -9,7 +11,8 @@ The expected proportion of individuals who carry the b allele but are not expect
 According to the Hardy-Weinberg equation p∧2 + 2pq + q∧2 = 1, where p is the frequency of dominant allele frequency, q is the frequency of recessive allele frequency, p∧2 is the frequency of the homozygous dominant allele, q∧2 is the frequency of the recessive allele, and 2pq is the frequency of the heterozygous allele.
 Given that q∧2=1/400, hence, q=0.05 and p=1-q=0.95.
 The frequency of the heterozygous allele is 2pq=2*0.05*0.95=38/400.
-The correct answer is (D)
+ANSWER: D
+
 Question: A Fe pellet of 0.056 g is first dissolved in 10 mL of hydrobromic acid HBr (0.1 M). The resulting solution is then titrated by KMnO4 (0.02 M). How many equivalence points are there?
 Choices:
 (A) Two points, 25 ml and 35 ml
@@ -30,7 +33,8 @@ Reaction 1: 		MnO4-   +  5Fe2+ + 8H+    → 	Mn2+	+    5Fe3+ + 4H2O
 Reaction 2: 		2MnO4-   +  10Br-   + 16H+    → 	2Mn2+	+    5Br2     + 8H2O
 So MnO4- will first react with Fe2+ with a stoichiometry of 1:5 so Veq1 will be 10 ml.
 Then when Fe2+ is used up, MnO4- will react with Br- with a stoichiometry of 2:10 then V added will be 25 ml so Veq2=25+10=35 ml.
-The correct answer is (A)
+ANSWER: A
+
 Question: Consider a quantum mechanical system containing a particle of mass $m$ moving in an istropic three dimensional potential of the form $V(r) = 1/2 m \omega^2 r^2$ corresponding to the acted force obeying Hooke’s law. Here, $\omega$ is the angular frequency of oscillation and $r$ is the radial distance of the particle from the origin in spherical polar coordinate. What is the value of energy of the third excited state, and how many linearly independent eigenfunctions are possible for the same energy eigenvalue?
 Choices:
 (A) 11 \pi^2 \hbar^2 / (2m r^2), 3
@@ -45,7 +49,8 @@ For third excited state n=3.
 Thus the corresponding energy is $(9/2)\hbar \omega$.
 The degeneracy of the state is $g_n= (n+1)(n+2)/2$.
 For n=3, degeneracy is (3+1)*(3+2)/2=4*5/2=10.
-The correct answer is (B)
+ANSWER: B
+
 Question: "Your overhear two chemists talking to each other as they leave a synthetic organic chemistry lab. One asks the other "So, how did it go?" The second chemist replies, "Not well - my compounds are on top of each other." What is the second chemist most likely referring to?"
 Choices:
 (A) The compounds they are working with have similar polarities.
@@ -55,7 +60,8 @@ Choices:
 Let's think step by step:
 "On top of each other" commonly refers to two compounds that have similar Rf values on chromatography (a common operation in synthetic chemistry).
 Similar Rf values arise for compounds with similar polarities.
-The correct answer is (A)
+ANSWER: A
+
 Question: Two people are playing the following game. A fair coin is tossed into the air. Person A says that in a single toss of the coin, the tail will come. So it's like the first shot or the third shot or the fifth shot. Person B says that the coin will come with a double toss. So like the second, fourth, sixth or eighth shot. Imagine this game played forever. What is the probability that person A wins this game?
 Choices:
 (A) 1/2
@@ -78,4 +84,5 @@ The solution for this series is as follows : a1/(1-r) where a1 is the first numb
 a1=1/2
 r=(1/2)^2=1/4
 So a1/(1-r)=(1/2)/(1-1/4)=(1/2)/(3/4)=2/3.
-The correct answer is (C)
+ANSWER: C
+"""