evalscope 0.15.1__py3-none-any.whl → 0.16.1__py3-none-any.whl

evalscope/benchmarks/data_adapter.py

@@ -5,7 +5,7 @@ from abc import ABC, abstractmethod
 from collections import defaultdict
 from typing import Any, Dict, List, Optional, Union
 
-from evalscope.benchmarks.utils import PromptData, preprocess_decorator
+from evalscope.benchmarks.utils import PromptData, load_file_with_extension, preprocess_decorator
 from evalscope.constants import DEFAULT_DATASET_CACHE_DIR, AnswerKeys, EvalType, HubType
 from evalscope.metrics import LLMJudge, metric_registry
 from evalscope.report import Report, ReportGenerator
@@ -15,6 +15,13 @@ logger = get_logger()
 
 
 class DataAdapter(ABC):
+    """
+    Data Adapter for the benchmark. You need to implement the following methods:
+        - gen_prompt
+        - get_gold_answer
+        - parse_pred_result
+        - match
+    """
 
     def __init__(self,
                  name: str,
@@ -31,30 +38,36 @@ class DataAdapter(ABC):
                  system_prompt: Optional[str] = None,
                  query_template: Optional[str] = None,
                  pretty_name: Optional[str] = None,
+                 description: Optional[str] = None,
                  **kwargs):
         """
-        Data Adapter for the benchmark. You need to implement the following methods:
-            - gen_prompt
-            - get_gold_answer
-            - parse_pred_result
-            - match
         Args:
             name: str, the name of the benchmark.
             dataset_id: str, the dataset id on ModelScope or local path for the benchmark.
+            model_adapter: str, the model adapter to use for the benchmark.
             subset_list: list of subset names for the dataset.
             metric_list: list, the metric list to evaluate the model on specific benchmark.
+            llm_as_a_judge: bool, whether to use LLM as a judge to evaluate the predicted answer against the gold answer.
+            output_types: list, the output types of the model adapter. Default: [model_adapter]
             few_shot_num: int, number of few-shot examples. Default: 0
             train_split: str, usually for few-shot examples. e.g. 'train'
             eval_split: str, the target eval split name. e.g. 'test'
             prompt_template: str, the prompt template for the benchmark,
                 e.g. for ARC, it is `The following are multiple choice questions, please output correct answer in
                     the form of A or B or C or D, do not output explanation:`
-        """
+            system_prompt: str, the system prompt for the benchmark, e.g. 'You are a helpful assistant.'
+            query_template: str, the query template for the benchmark, e.g. 'Please answer the following question: {}'
+            pretty_name: str, the pretty name of the benchmark, e.g. 'ARC Challenge Set'.
+            description: str, the description of the benchmark,
+                e.g. 'ARC Challenge Set is a benchmark for evaluating reasoning abilities of models on science questions.'
+        """  # noqa: E501
         self.name = name
         self.dataset_id = dataset_id
         self.model_adapter = model_adapter
         self.subset_list = subset_list
         self.metric_list = metric_list
+        self.llm_as_a_judge = llm_as_a_judge
+        self.output_types = output_types or [model_adapter]
         self.few_shot_num = few_shot_num
         self.train_split = train_split
         self.eval_split = eval_split
@@ -62,9 +75,8 @@ class DataAdapter(ABC):
         self.system_prompt = system_prompt
         self.query_template = query_template
         self.pretty_name = pretty_name
+        self.description = description
         self.config_kwargs = kwargs
-        self.output_types = output_types or [model_adapter]
-        self.llm_as_a_judge = llm_as_a_judge
         self.category_map = kwargs.get('category_map', {})
         self.choices = kwargs.get('choices', None)
 
@@ -156,6 +168,49 @@ class DataAdapter(ABC):
         """
         return self.load_from_hub(dataset_name_or_path, subset_list, work_dir, **kwargs)
 
+    def load_with_snapshot(self,
+                           file_structure: Dict[str, List[str]],
+                           dataset_name_or_path: str = None,
+                           subset_list: list = None,
+                           work_dir: Optional[str] = DEFAULT_DATASET_CACHE_DIR,
+                           **kwargs) -> dict:
+        """
+        For datasets that cannot be correctly loaded using MsDataset, utilize snapshot downloading to load the data.
+        This feature supports both remote and local datasets.
+
+        Args:
+            file_structure: dict, the file structure of the dataset, e.g. {'subset_name': ['file1.jsonl', 'file2.jsonl']}.
+            dataset_name_or_path: str, the dataset id on ModelScope or local path for the benchmark.
+            subset_list: list of subset names for the dataset.
+            work_dir: str, the working directory to store the dataset.
+        Returns: {'subset_name': {'eval': eval_dataset}}
+        """  # noqa: E501
+        dataset_name_or_path = os.path.expanduser(dataset_name_or_path or self.dataset_id)
+        subset_list = subset_list or self.subset_list
+
+        # Try to load dataset from local disk
+        if os.path.exists(dataset_name_or_path):
+            logger.info(f'Loading dataset from {dataset_name_or_path}')
+            dataset_path = dataset_name_or_path
+        else:
+            from modelscope import dataset_snapshot_download
+
+            # Load dataset from remote
+            logger.info(f'Loading dataset from modelscope: > dataset_name: {dataset_name_or_path}')
+            # flatten file structure
+            file_names = [file for sub_files in file_structure.values() for file in sub_files]
+            # download dataset snapshot
+            dataset_path = dataset_snapshot_download(
+                dataset_name_or_path, cache_dir=work_dir, allow_file_pattern=file_names)
+        # read and process files
+        data_dict = defaultdict(dict)
+        for sub_name in subset_list:
+            file_paths = [os.path.join(dataset_path, file_name) for file_name in file_structure[sub_name]]
+            # not train split, only eval split
+            data_dict[sub_name][self.eval_split] = load_file_with_extension(file_paths)
+
+        return data_dict
+
     def reformat_subset(self, data_dict: dict, subset_key: str, format: str = '{}') -> dict:
         """
         Reformat the dataset subset with subset_key and format.
@@ -249,7 +304,7 @@ class DataAdapter(ABC):
     def compute_dict_metric(self, review_res_list: Union[List[dict], List[List[dict]]],
                             **kwargs) -> Dict[str, List[float]]:
         """
-        compute weighted mean of the bleu score of all samples
+        compute weighted mean of score of all samples
 
         Args:
             review_res_list: [score1, score2, ...]
@@ -258,7 +313,7 @@ class DataAdapter(ABC):
             avg_res: Dict[str, List[float]]
 
         """
-        if isinstance(review_res_list[0], list):
+        if len(review_res_list) > 0 and isinstance(review_res_list[0], list):
             review_res_list = [item for sublist in review_res_list for item in sublist]
 
         items = defaultdict(list)
@@ -270,7 +325,7 @@ class DataAdapter(ABC):
                 items['AverageAccuracy'].append(scores)
         return items
 
-    def gen_report(self, subset_score_map: dict, report_name: str = None, **kwargs) -> Report:
+    def gen_report(self, subset_score_map: dict, model_name: str, **kwargs) -> Report:
         """
         Generate report for the evaluation results for all subsets.
 
@@ -278,7 +333,7 @@ class DataAdapter(ABC):
             subset_score_map: The subset-score map.
                 e.g. {subset_name: [{'metric_name': 'AverageAccuracy', 'score': 0.3389, 'num': 100}]}
 
-            report_name: str, the user-defined report name. Default: None
+            model_name: The evaluation model name.
 
         Returns: The evaluation report.
 
@@ -312,9 +367,17 @@ class DataAdapter(ABC):
             "model_name": "qwen2.5"
         }
         """  # noqa: E501
-        kwargs['category_map'] = self.category_map
-        kwargs['metric_list'] = self.metric_list
-        return ReportGenerator.gen_report(subset_score_map, report_name, **kwargs)
+        return ReportGenerator.gen_report(subset_score_map, model_name, data_adapter=self, **kwargs)
+
+    def post_process_report(self, report: Report, **kwargs):
+        """
+        Post-process the report after generation. Draw a chart, save to file, etc.
+        This method can be overridden to customize the report format or content.
+
+        Args:
+            report (Report): The generated report.
+        """
+        pass
 
     def gen_prompt_data(self,
                         prompt: str,
@@ -322,14 +385,33 @@ class DataAdapter(ABC):
                         choices: Optional[List[str]] = None,
                         index: Optional[Union[int, str]] = None,
                         id: Optional[Union[int, str]] = None,
+                        messages: Optional[List[dict]] = None,
                         **kwargs) -> dict:
+        """
+        Generates a dictionary representation of prompt data for evaluation or inference.
+
+        Args:
+            prompt (str): The main prompt or input text. Can also be a list of prompts.
+            system_prompt (Optional[str], optional): An optional system-level prompt to provide context or instructions. Defaults to None.
+            choices (Optional[List[str]], optional): A list of possible choices for multi-choice tasks.
+                If not provided, uses self.choices. Defaults to None.
+            index (Optional[Union[int, str]], optional): An optional index or identifier for the prompt.
+                Defaults to 0 if not provided. Defaults to None.
+            id (Optional[Union[int, str]], optional): An optional unique identifier for the prompt data. Defaults to None.
+            messages (Optional[List[dict]], optional): An optional list of message dictionaries, typically for chat-based prompts. Defaults to None.
+                If messages is provided, it will be used as the prompt data instead of the prompt string.
+
+        Returns:
+            dict: A dictionary representation of the prompt data, suitable for further processing or model input.
+        """  # noqa: E501
         data = [prompt] if not isinstance(prompt, list) else prompt
         prompt_data = PromptData(
             data=data,
             multi_choices=choices or self.choices,
             system_prompt=system_prompt or self.system_prompt,
             index=index or 0,
-            id=id)
+            id=id,
+            messages=messages)
         return prompt_data.to_dict()
 
     def gen_prompt(self, input_d: dict, subset_name: str, few_shot_list: list, **kwargs) -> Any:
@@ -414,7 +496,8 @@ class DataAdapter(ABC):
 
         # Extract question from raw_input if available
         raw_input = kwargs.get('raw_input', {})
-        question_keys = ['question', 'prompt', 'query', 'problem']
+        question_keys = ['question', 'Question', 'prompt', 'Prompt', 'query', 'Query', 'problem', 'Problem']
+        # Find the first non-empty question key in raw_input
         question = next((raw_input.get(key) for key in question_keys if raw_input.get(key)), None)
 
         # Request judge and obtain score

evalscope/benchmarks/docmath/docmath_adapter.py

@@ -0,0 +1,84 @@
+from evalscope.benchmarks import Benchmark, DataAdapter
+from evalscope.constants import EvalType
+from evalscope.metrics import LLMJudge
+
+TEMPLATE_0SHOT = """Please read the following text and answer the question below.
+
+<text>
+{context}
+</text>
+
+{question}
+
+Format your response as follows: "Therefore, the answer is (insert answer here)"."""
+
+
+@Benchmark.register(
+    name='docmath',
+    pretty_name='DocMath',
+    description=
+    'DocMath-Eval is a comprehensive benchmark focused on numerical reasoning within specialized domains. It requires the model to comprehend long and specialized documents and perform numerical reasoning to answer the given question.',  # noqa: E501
+    dataset_id='yale-nlp/DocMath-Eval',
+    metric_list=['AverageAccuracy'],
+    subset_list=['complong_testmini', 'compshort_testmini', 'simplong_testmini', 'simpshort_testmini'],
+    few_shot_num=0,
+    train_split=None,
+    eval_split='test',
+    prompt_template=TEMPLATE_0SHOT,
+)
+class DocMathAdapter(DataAdapter):
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    def load(self, **kwargs):
+        # default load mini test
+        kwargs['split_as_subset'] = True
+        data_dict = super().load(**kwargs)
+        return data_dict
+
+    def gen_prompt(self, input_d: dict, subset_name: str, few_shot_list: list, **kwargs) -> dict:
+        """
+        Generate model prompt from input data.
+        """
+        context = context = '\n'.join(input_d['paragraphs'])
+        question = input_d['question']
+        prompt = self.prompt_template.format(context=context, question=question)
+        return self.gen_prompt_data(prompt)
+
+    def get_gold_answer(self, input_d: dict) -> str:
+        """
+        Parse the raw input labels (gold).
+        """
+        return input_d['ground_truth']
+
+    def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = EvalType.CHECKPOINT) -> str:
+        """
+        Parse the predicted result and extract proper answer.
+        """
+        from .utils import extract_answer
+
+        extracted_answer = extract_answer(result)
+        return extracted_answer
+
+    def match(self, gold: str, pred: str) -> float:
+        """
+        Match the gold answer and the predicted answer.
+        """
+        from .utils import get_acc
+
+        return get_acc(prediction=pred, gt=gold)
+
+    def llm_match(self, gold: str, pred: str, judge: LLMJudge, **kwargs) -> float:
+        from .utils import GENERAL_ORM_PROMPT, ORM_USER_TEMPLATE
+
+        raw_input = kwargs.get('raw_input', None)
+        question = raw_input['question']
+        # get grading response
+        prompt = ORM_USER_TEMPLATE.format(problem=question, answer_1=gold, answer_2=pred)
+        orm_response = judge(prompt=prompt, system_prompt=GENERAL_ORM_PROMPT)
+        # parse grading response
+        if 'YES' in orm_response:
+            return 1.0
+        else:
+            return 0.0

evalscope/benchmarks/docmath/utils.py

@@ -0,0 +1,220 @@
+import math
+import numpy as np
+import re
+from sympy import Rational
+
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+GENERAL_ORM_PROMPT = """You are an expert in verifying if two answers are the same.
+Your input is a problem and two answers, Answer 1 and Answer 2. You need to check if they are equivalent.
+Your task is to determine if two answers are equivalent, without attempting to solve the original problem.
+Compare the answers to verify they represent identical values or meaning, even when written in different forms or notations.
+
+Your output must follow the following format:
+1) Provide an explanation for why the answers are equivalent or not.
+2) Then provide your final answer in the form of: [[YES]] or [[NO]]
+"""  # noqa: E501
+
+ORM_USER_TEMPLATE = """
+Problem: {problem}
+Answer 1: {answer_1}
+Answer 2: {answer_2}
+"""
+
+
+def round_up_to_decimal(number, decimals):
+    factor = 10**decimals
+    return math.ceil(number * factor) / factor
+
+
+def is_number(string):
+    pattern = r'^[-+]?(\d{1,3}(,\d{3})*|(\d+))(\.\d+)?$'
+    match = re.match(pattern, string)
+    return bool(match)
+
+
+def is_scientific_number(string):
+    pattern = r'^[-+]?\d+(\.\d+)?e[-]?\d+$'
+    match = re.match(pattern, string)
+    return bool(match)
+
+
+def normalize(prediction: str):
+    # Preprocessing the string [Stage 1]
+    prediction = prediction.strip()
+    prediction = prediction.rstrip('.')
+    if not isinstance(prediction, str):
+        prediction = str(prediction) if prediction is not None else '0'
+
+    for money in ['£', '€', '¥', 'million', 'billion', 'thousand', 'US', 'USD', 'RMB']:
+        prediction = prediction.replace(money, '')
+
+    # Replace special tokens
+    if '=' in prediction:
+        prediction = prediction.split('=')[-1].strip()
+    if '≈' in prediction:
+        prediction = prediction.split('≈')[-1].strip()
+    if '`' in prediction:
+        prediction = prediction.replace('`', '')
+    if '%' in prediction:
+        prediction = prediction.replace('%', '')
+    if '$' in prediction:
+        prediction = prediction.replace('$', '')
+    if '°' in prediction:
+        prediction = prediction.replace('°', '')
+
+    # Detect the boolean keyword in the generation
+    if prediction in ['true', 'yes', 'false', 'no']:
+        if prediction == 'true' or prediction == 'yes':
+            prediction = 'True'
+        else:
+            prediction = 'False'
+    if 'True' in prediction or 'False' in prediction:
+        prediction = 'True' if 'True' in prediction else 'False'
+
+    # Detect the approximation keyword
+    if 'approximately' in prediction:
+        prediction = prediction.replace('approximately', '').strip()
+    if ' or ' in prediction:
+        prediction = prediction.split(' or ')[0]
+
+    # Drop the units before and after the number
+    if re.match(r'[-+]?(?:[\d,]*\.*\d+) [^0-9 ]+$', prediction):
+        prediction = re.search(r'([-+]?(?:[\d,]*\.*\d+)) [^0-9 ]+$', prediction).group(1)
+    if re.match(r'[^0-9 ]+ [-+]?(?:[\d,]*\.*\d+)$', prediction):
+        prediction = re.search(r'[^0-9 ]+ ([-+]?(?:[\d,]*\.*\d+))$', prediction).group(1)
+    if re.match(r'[-+]?(?:[\d,]*\.*\d+)[^\d]{1,2}$', prediction):
+        prediction = re.search(r'([-+]?(?:[\d,]*\.*\d+))[^\d]{1,2}$', prediction).group(1)
+    if re.match(r'[^-+\d]{1,2}(?:[\d,]*\.*\d+)$', prediction):
+        prediction = re.search(r'[^-+\d]{1,2}((?:[\d,]*\.*\d+))$', prediction).group(1)
+
+    # Preprocessing the number [Stage 1]
+    if '10^' in prediction:
+        prediction = re.sub(r'10\^(-?\d+)', r'math.pow(10, \1)', prediction)
+    if ' x ' in prediction:
+        prediction = prediction.replace(' x ', '*')
+    if ' × ' in prediction:
+        prediction = prediction.replace(' × ', '*')
+    if is_number(prediction):
+        prediction = prediction.replace(',', '')
+
+    # Preprocessing the option [Stage 3]
+    if '(a)' in prediction or '(b)' in prediction or '(c)' in prediction or '(d)' in prediction:
+        prediction = '"' + re.search(r'\([a-d]\)', prediction).group(0) + '"'
+
+    # If the prediction is empty, use dummy '0'
+    if not prediction:
+        prediction = '0'
+
+    # Converting the string answer to a number/list/bool/option
+    try:
+        prediction = eval(prediction)
+    except Exception:
+        # TO CHECK
+        prediction = 0
+
+    # Performing common type conversion
+    if isinstance(prediction, (set, tuple)):
+        prediction = list(prediction)
+        if isinstance(prediction[0], complex):
+            prediction = [tmp.real for tmp in prediction]
+        elif isinstance(prediction[0], Rational):
+            prediction = [float(tmp) for tmp in prediction]
+    elif isinstance(prediction, np.ndarray):
+        prediction = prediction.tolist()
+    else:
+        if isinstance(prediction, complex):
+            prediction = prediction.real
+        elif isinstance(prediction, Rational):
+            prediction = float(prediction)
+
+    return prediction
+
+
+def extract_answer(response: str):
+    """Parses the final answer from the model's response text.
+
+    Args:
+        response: Text extracted from the model's response
+
+    Returns:
+        The final answer as a numeric value (string), or None if not found
+    """
+    # Remove any asterisks or other unwanted characters
+    response = response.replace('*', '')
+    response = response.replace('(', '')
+    response = response.replace(')', '')
+
+    # Search for the pattern 'the answer is {final answer}.'
+    match = re.search(r'the answer is (\=?\≈?\`?\%?\$?\°?\£?\€?\¥?-?[0-9\.,]+)', response, re.IGNORECASE)
+
+    if match:
+        # Remove commas from the matched number (if any)
+        res = match.group(1).replace(',', '').rstrip('.')
+        return res
+    else:
+        return response
+
+
+def within_eps(pred: float, gt: float):
+    eps = abs(gt) * 0.0015
+    if pred >= gt - eps and pred <= gt + eps:
+        return True
+    else:
+        return False
+
+
+def compare_two_numbers(p, gt):
+    if isinstance(p, int) or isinstance(p, float):
+        pass
+    elif isinstance(p, list) or isinstance(p, bool) or isinstance(p, str):
+        return False
+    elif isinstance(p, tuple) or isinstance(p, complex) or isinstance(p, dict):
+        return False
+    else:
+        raise ValueError(p)
+
+    v1, v2 = max(abs(gt), abs(p)), min(abs(gt), abs(p))
+    if (v1 != 0 and v2 != 0) and int(math.log10(v1 / v2)) == math.log10(v1 / v2):
+        return True
+
+    if v2 <= v1 / 50 and within_eps(pred=v2 * 100, gt=v1):
+        return True
+    elif v2 <= v1 / 500 and within_eps(pred=v2 * 1000, gt=v1):
+        return True
+    elif v2 <= v1 / 50000 and within_eps(pred=v2 * 100000, gt=v1):
+        return True
+
+    if round_up_to_decimal(v1, 2) == round_up_to_decimal(v2, 2):
+        return True
+
+    return within_eps(pred=p, gt=gt)
+
+
+def get_acc(prediction, gt, cot=True):
+    try:
+        if cot:
+            prediction = normalize(prediction)
+        else:
+            prediction = float(prediction)
+
+        answer_type = type(gt).__name__
+        assert answer_type in ['int', 'float', 'float64', 'bool'], answer_type
+        if isinstance(prediction, (str, int, float, bool)) or isinstance(prediction, list):
+            # Comparing prediction against the reference
+            if answer_type in ['bool']:
+                acc = int(prediction == gt)
+            elif answer_type == 'int':
+                acc = int(compare_two_numbers(prediction, gt))
+            elif answer_type == 'float' or answer_type == 'float64':
+                acc = int(compare_two_numbers(prediction, gt))
+            else:
+                acc = 0
+        else:
+            acc = 0
+            logger.error('Error: ', prediction, type(prediction))
+        return acc
+    except Exception:
+        return 0

evalscope/benchmarks/drop/drop_adapter.py

@@ -0,0 +1,133 @@
+import re
+from typing import List
+
+from evalscope.benchmarks import Benchmark, DataAdapter
+from evalscope.constants import EvalType
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+DROP_EXAMPLES = '''Some examples of passages and Q&A are provided below.
+
+# Examples
+---
+Passage: Trunajaya rebellion  or Trunajaya War was the ultimately unsuccessful rebellion waged by the Madurese prince Trunajaya and fighters from Makassar against the Mataram Sultanate and its Dutch East India Company  supporters in Java  during the 1670s. The rebellion was initially successful: the rebels defeated the royal army at Gegodog , captured most of the Javanese north coast, and took the Mataram capital Plered . King Amangkurat I died during the retreat of the royal court. His son and successor, Amangkurat II, requested help from the VOC in exchange for financial remuneration and geopolitical concessions. The VOC\'s subsequent involvement turned the tide of the war. VOC and Mataram forces recovered lost territories and overran Trunajaya\'s new capital at Kediri . However, the rebellion continued until the capture of Trunajaya at the end of 1679, and the defeat, death, or surrender of the other rebel leaders . Trunajaya was killed by Amangkurat II personally in 1680 while a prisoner of the VOC. After his father\'s death in 1677, Amangkurat II also faced rival claims to the throne. The most serious rival was his brother Pangeran Puger, who took the capital Plered in 1677 and did not surrender until 1681.
+Question: How many years was it between Trunajaya\'s capture and his death while prisoner of the VOC?
+Answer:  1
+
+---
+Passage: Led by former Giant Kurt Warner, the defending NFC champions took the field at Giants Stadium against a Giants team still reeling from their bad loss in New Orleans. The Giants scored first, sending Jacobs in for a 4-yard touchdown run following a Terrell Thomas interception. Later, Arizona running back Beanie Wells scored his first career touchdown on a 13-yard rush. Manning responded by throwing a 62-yard touchdown to Nicks for his longest reception of the year. In the second half, the Cardinals\' Tim Hightower and Jason Wright scored touchdowns. But it was turnovers that decided this game; Manning\'s 3 interceptions were as many as he had thrown all season. The Giants scored only 3 points in the second half, ending the game on an interception to Antrel Rolle. The Giants notable streak of 38 consecutive starts by the same offensive line unit was ended here, as offensive tackle Kareem McKenzie missed the game with a groin injury. McKenzie returned the following week.
+Question: Which player made the first score of the game?
+Answer:  Jacobs
+
+---
+Passage: Hoping to rebound from their road loss to the Bills, the Chargers flew to Wembley Stadium for the 2008 International Series game with the New Orleans Saints. In the first quarter, San Diego trailed early as kicker Taylor Mehlhaff got a 23-yard field goal.  The \'Bolts would respond with kicker Nate Kaeding getting a 33-yard field goal.  In the second quarter, New Orleans regained the lead as QB Drew Brees (a former Charger) completed a 12-yard TD pass to WR Devery Henderson (with a failed PAT) and RB Deuce McAllister getting a 1-yard TD run.  San Diego answered as QB Philip Rivers completed a 12-yard TD pass to RB LaDainian Tomlinson, but the Saints replied with Brees completing a 30-yard TD pass to WR Lance Moore.  The Chargers closed out the half with Rivers completing a 12-yard TD pass to TE Antonio Gates. In the third quarter, New Orleans increased its lead Brees completing a 1-yard TD pass to TE Mark Campbell, after a very controversial Pass interference call on cornerback Cletis Gordon put the Saints on the 1-yard line.  The \'Bolts would answer with Kaeding getting a 24-yard field goal.  In the fourth quarter, the Saints continued to build its lead as FB Mike Karney got a 1-yard TD run.  San Diego tried to rally as Kaeding nailed a 31-yard field goal, Rivers completed a 14-yard TD pass to WR Vincent Jackson, and Brees giving the \'Bolts a safety via an incomplete pass thrown into the back of his own endzone.  However, New Orleans\' defense stiffened for the win. With the loss, the Chargers went into their bye week at 3-5.
+Question: How many total yards of touchdown passes did Drew Brees make?
+Answer:  43
+
+'''  # noqa: E501
+
+
+@Benchmark.register(
+    name='drop',
+    pretty_name='DROP',
+    dataset_id='AI-ModelScope/DROP',
+    metric_list=['AverageAccuracy'],
+    few_shot_num=0,
+    train_split=None,
+    eval_split='validation',
+    prompt_template=
+    'You will be asked to read a passage and answer a question.{drop_examples}# Your Task\n\n---\n{query}\n\nThink step by step, then write a line of the form "Answer: $ANSWER" at the end of your response.',  # noqa: E501
+)
+class DROPAdapter(DataAdapter):
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+        few_shot_num = kwargs.get('few_shot_num', 0)
+        if few_shot_num != 0:
+            self.few_shot_num = 3
+            logger.info(f'Few shot num is set to {self.few_shot_num} for DROP dataset by system.')
+        else:
+            self.few_shot_num = 0
+
+    def gen_prompt(self, input_d: dict, subset_name: str, few_shot_list: list, **kwargs) -> dict:
+        """
+        Generate model prompt from input data.
+        """
+        drop_examples = '' if self.few_shot_num == 0 else DROP_EXAMPLES
+        query = f"Passage: {input_d['passage']}\nQuestion: {input_d['question']}"
+        prompt = self.prompt_template.format(
+            drop_examples=drop_examples,
+            query=query,
+        )
+        return self.gen_prompt_data(prompt)
+
+    def get_gold_answer(self, input_d: dict) -> List[str]:
+        """
+        Parse the raw input labels (gold).
+        """
+
+        def _flatten_validated_answers(validated_answers):
+            """Flattens a dict of lists of validated answers.
+            {"number": ['1', '8'], ...}
+            -> [{"number": ['1'], ...}, {"number": ['8'], ...}]
+            """
+            valid_answers = []
+            for i in range(len(validated_answers['number'])):
+                valid_answers.append({
+                    'number': validated_answers['number'][i],
+                    'date': validated_answers['date'][i],
+                    'spans': validated_answers['spans'][i],
+                })
+            return valid_answers
+
+        answers = []
+        answers_set = set()
+        candidates = [input_d['answer']] + _flatten_validated_answers(input_d['validated_answers'])
+        for candidate in candidates:
+            answer = DROPAdapter.parse_answer(candidate)
+            if answer in answers_set:
+                continue
+            answers_set.add(answer)
+            answers.append(answer)
+        return answers
+
+    def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = EvalType.CHECKPOINT) -> str:
+        """
+        Parse the predicted result and extract proper answer.
+        """
+        match = re.search(r'(?i)Answer\s*:\s*([^\n]+)', result)
+        extracted_answer = match.group(1) if match else result
+        return extracted_answer
+
+    def match(self, gold: List[str], pred: str) -> float:
+        """
+        Match the gold answer and the predicted answer.
+        """
+        from .utils import _answer_to_bags
+
+        max_em = 0
+        for gold_answer in gold:
+            # Convert the answers to bags of answers
+            predicted_bags = _answer_to_bags(pred)
+            gold_bags = _answer_to_bags(gold_answer)
+
+            if set(predicted_bags[0]) == set(gold_bags[0]) and len(predicted_bags[0]) == len(gold_bags[0]):
+                exact_match = 1.0
+            else:
+                exact_match = 0.0
+            # Check if the answer is empty
+            if gold_answer[0].strip():
+                max_em = max(max_em, exact_match)
+
+        return max_em
+
+    @staticmethod
+    def parse_answer(answer):
+        # NOTE: Everything is returned as a tuple for uniformity and hashability.
+        if answer['number'] != '':
+            return (str(answer['number']), )
+        if answer['spans'] != []:
+            return tuple(answer['spans'])
+        return (' '.join([answer['date']['day'], answer['date']['month'], answer['date']['year']]).strip(), )