evalscope 0.17.1__py3-none-any.whl → 1.0.1__py3-none-any.whl

evalscope/benchmarks/general_arena/utils.py

@@ -7,44 +7,39 @@ from collections import defaultdict
 from sklearn.linear_model import LogisticRegression
 from tqdm import tqdm
 
+from evalscope.api.evaluator import ReviewResult
 from evalscope.utils.logger import get_logger
 
 logger = get_logger()
 
 
-def process_review_item(review_item: dict) -> dict:
+def process_review_item(review_result: ReviewResult) -> list:
     """
-    Process a single review item to extract relevant information.
+    Process a ReviewResult object to extract relevant information.
 
     Args:
-        review_item (dict): The review item to process.
+        review_result: ReviewResult object or dict (for backward compatibility)
 
     Returns:
-        dict: Processed review item with necessary information.
+        list: List of processed review items with necessary information.
     """
-    res = []
-    raw_input = review_item['raw_input']
-    sample_index = review_item['index']
-    question_keys = ['question', 'Question', 'prompt', 'Prompt', 'query', 'Query', 'problem', 'Problem']
-    # Find the first non-empty question key in raw_input
-    question = next((raw_input.get(key) for key in question_keys if raw_input.get(key)), None)
-    for choice_index, choice in enumerate(review_item['choices']):
-        raw_pred_answer = choice['message']['content']
-        parsed_gold_answer = choice['review']['gold']
-        parsed_pred_answer = choice['review']['pred']
-        score = choice['review']['result']
-        raw_d = {
-            'Index': f'{sample_index}_{choice_index}',
-            'Input': raw_input,
-            'Question': question if question else '*No Question*',
-            'Generated': raw_pred_answer,
-            'Gold': parsed_gold_answer if parsed_gold_answer != raw_input else '*Same as Input*',
-            'Pred': parsed_pred_answer,
-            'Score': score,
-        }
-        res.append(raw_d)
 
-    return res
+    # New format using ReviewResult
+    sample_score = review_result.sample_score
+    prediction = sample_score.score.prediction
+    target = review_result.target
+    extracted_prediction = sample_score.score.extracted_prediction
+
+    raw_d = {
+        'Index': str(review_result.index),
+        'Input': review_result.input,
+        'Question': review_result.input,  # Use input as question
+        'Generated': prediction if prediction != extracted_prediction else extracted_prediction,
+        'Gold': target,
+        'Pred': extracted_prediction,
+        'Score': sample_score.score.model_dump(exclude_none=True),
+    }
+    return [raw_d]
 
 
 def post_process_result(completion):
@@ -179,7 +174,8 @@ def compute_mle_elo(df, scale=400, base=10, init_rating=1000, baseline_model='gp
         return elo_scores.sort_values(ascending=False)
 
     lr = LogisticRegression(
-        fit_intercept=False, penalty=None, tol=1e-8)  # May need to set a small value when not use GPT4 as judge model
+        fit_intercept=False, penalty=None, tol=1e-8
+    )  # May need to set a small value when not use GPT4 as judge model
     lr.fit(X, Y)
 
     elo_scores = scale * lr.coef_[0] + init_rating

evalscope/benchmarks/general_mcq/general_mcq_adapter.py

@@ -2,118 +2,57 @@
 import os
 from collections import defaultdict
 
-from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.constants import EvalType, OutputType
-from evalscope.metrics import exact_match
-from evalscope.metrics.completion_parsers import ResponseParser
+from evalscope.api.benchmark import BenchmarkMeta, MultiChoiceAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
 from evalscope.utils.io_utils import csv_to_list, jsonl_to_list
 from evalscope.utils.logger import get_logger
+from evalscope.utils.multi_choices import MultipleChoiceTemplate
 
 # flake8: noqa
 
 logger = get_logger()
 
 
-@Benchmark.register(
-    name='general_mcq',
-    pretty_name='General-MCQ',
-    description='A general multiple-choice question answering dataset for custom evaluation. '
-    'For detailed instructions on how to use this benchmark, please refer to the [User Guide](https://evalscope.readthedocs.io/zh-cn/latest/advanced_guides/custom_dataset/llm.html#mcq).',
-    tags=['MCQ', 'Custom'],
-    dataset_id='general_mcq',
-    model_adapter=OutputType.GENERATION,
-    output_types=[OutputType.MULTIPLE_CHOICE, OutputType.GENERATION],
-    subset_list=['default'],
-    metric_list=['AverageAccuracy'],
-    few_shot_num=0,
-    train_split='dev',
-    eval_split='val',
-    prompt_template='请回答问题，并选出其中的正确答案。你的回答的最后一行应该是这样的格式：“答案是：LETTER”（不带引号），其中 LETTER 是 A、B、C、D 中的一个。\n{query}',
-    query_template='问题：{question}\n{choices}\n答案: {answer}\n\n')
-class GeneralMCQAdapter(DataAdapter):
+@register_benchmark(
+    BenchmarkMeta(
+        name='general_mcq',
+        pretty_name='General-MCQ',
+        description='A general multiple-choice question answering dataset for custom evaluation. '
+        'For detailed instructions on how to use this benchmark, please refer to the [User Guide](https://evalscope.readthedocs.io/zh-cn/latest/advanced_guides/custom_dataset/llm.html#mcq).',
+        tags=[Tags.MULTIPLE_CHOICE, Tags.CUSTOM],
+        dataset_id='general_mcq',
+        subset_list=['default'],
+        metric_list=['acc'],
+        few_shot_num=0,
+        train_split='dev',
+        eval_split='val',
+        prompt_template=MultipleChoiceTemplate.CHINESE_SINGLE_ANSWER_TEMPLATE,
+    )
+)
+class GeneralMCQAdapter(MultiChoiceAdapter):
 
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
 
         self.choices = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J']
 
-    def load_from_disk(self, dataset_name_or_path, subset_list, work_dir, **kwargs) -> dict:
-        data_dict = defaultdict(dict)
-        for subset_name in subset_list:
-            for split_name in [self.train_split, self.eval_split]:
-                # Check for files with different extensions
-                for ext, loader in [('.jsonl', jsonl_to_list), ('.csv', csv_to_list)]:
-                    if os.path.exists(dataset_name_or_path):
-                        file_path = os.path.join(dataset_name_or_path, f'{subset_name}_{split_name}{ext}')
-                    else:
-                        file_path = os.path.join(work_dir, dataset_name_or_path, f'{subset_name}_{split_name}{ext}')
-
-                    if os.path.exists(file_path):
-                        data_dict[subset_name][split_name] = loader(file_path)
-                        break  # Stop checking other extensions once a file is found
-
-        return dict(data_dict)
-
-    def gen_prompt(self, input_d: dict, subset_name: str, few_shot_list: list, **kwargs) -> dict:
-        """
-        Generate model prompt from raw input, unify the prompt format for C-Eval benchmark.
-
-        Args:
-            input_d (dict): The raw input. A single data format of the C-Eval:
-
-            {'id': 0,
-            'question': '下列关于税法基本原则的表述中，不正确的是____。',
-            'A': '税收法定原则包括税收要件法定原则和税务合法性原则',
-            'B': '税收公平原则源于法律上的平等性原则',
-            'C': '税收效率原则包含经济效率和行政效率两个方面',
-            'D': '税务机关按法定程序依法征税，可以自由做出减征、停征或免征税款的决定',
-            'answer': 'D'}
-
-        Returns:
-            {'data': ['prompt ...']}
-        """
-
-        few_shot_prompts = [self._format_example(input_d=sample, include_answer=True) for sample in few_shot_list]
-
-        if len(few_shot_prompts) > 0:
-            context: str = '\n'.join(few_shot_prompts) + '\n'
-        else:
-            context = ''
-        context = context.strip() + self._format_example(input_d=input_d, include_answer=False)
-
-        full_prompt = self.prompt_template.format(query=context)
-
-        return self.gen_prompt_data(full_prompt)
-
-    def get_gold_answer(self, input_d: dict) -> str:
-        # Get the gold choice
-        return input_d.get('answer', '')
-
-    def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = EvalType.CHECKPOINT) -> str:
-        """
-        Parse the model output to get the answer. Could be the best choice index.
-
-        Args:
-            result: Predicted answer from the model. Usually a string for chat.
-            raw_input_d (dict): The raw input. Depending on the dataset.
-            eval_type: `checkpoint` or `service` or `custom`. Default is `checkpoint`.
-
-        Returns:
-            The parsed answer. Depending on the dataset. Usually a string for chat.
-        """
-        if self.model_adapter == OutputType.MULTIPLE_CHOICE:
-            return result
-        else:
-            return ResponseParser.parse_first_option_with_choices(text=result, options=self.choices)
-
-    def match(self, gold: str, pred: str) -> float:
-        return exact_match(gold=gold, pred=pred)
-
-    def _format_example(self, input_d: dict, include_answer=True):
-        choices_str = '\n'.join([f'{choice}. {input_d[choice]}' for choice in self.choices if choice in input_d])
-
-        if include_answer:
-            return self.query_template.format(
-                question=input_d['question'], choices=choices_str, answer=input_d['answer'])
-        else:
-            return self.query_template.format(question=input_d['question'], choices=choices_str, answer='').rstrip()
+    def load_from_disk(self, **kwargs):
+        return super().load_from_disk(use_local_loader=True)
+
+    def record_to_sample(self, record) -> Sample:
+        # Extract choices from the record (A, B, C, D, etc.)
+        choices = []
+        for choice_key in self.choices:
+            if choice_key in record:
+                choices.append(record[choice_key])
+            else:
+                break  # Stop when we reach a choice key that doesn't exist
+
+        return Sample(
+            input=record['question'],
+            choices=choices,
+            target=record['answer'],
+            metadata={'id': record.get('id', 'unknown')},
+        )

evalscope/benchmarks/general_qa/general_qa_adapter.py

@@ -1,155 +1,94 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
-import os.path
-from collections import defaultdict
-from typing import List, Optional, Union
-
-from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.metrics import mean
-from evalscope.utils.io_utils import jsonl_to_list
+from typing import Any, Dict, List, Optional, Union
+
+from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.evaluator import TaskState
+from evalscope.api.messages import ChatMessageSystem, ChatMessageUser, dict_to_chat_message
+from evalscope.api.metric import Score
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
 from evalscope.utils.logger import get_logger
 
 logger = get_logger()
 
-
-@Benchmark.register(
-    name='general_qa',
-    pretty_name='General-QA',
-    description='A general question answering dataset for custom evaluation. '
-    'For detailed instructions on how to use this benchmark, please refer to the [User Guide](https://evalscope.readthedocs.io/zh-cn/latest/advanced_guides/custom_dataset/llm.html#qa).',  # noqa: E501
-    tags=['QA', 'Custom'],
-    dataset_id='general_qa',
-    subset_list=['default'],
-    metric_list=['AverageBLEU', 'AverageRouge'],
-    few_shot_num=0,
-    train_split=None,
-    eval_split='test',
-    prompt_template='请回答问题\n{query}',
+PROMPT_TEMPLATE = '请回答问题\n{question}'
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='general_qa',
+        pretty_name='General-QA',
+        description='A general question answering dataset for custom evaluation. '
+        'For detailed instructions on how to use this benchmark, please refer to the [User Guide](https://evalscope.readthedocs.io/zh-cn/latest/advanced_guides/custom_dataset/llm.html#qa).',  # noqa: E501
+        tags=[Tags.QA, Tags.CUSTOM],
+        dataset_id='general_qa',
+        metric_list=['BLEU', 'Rouge'],
+        few_shot_num=0,
+        train_split=None,
+        eval_split='test',
+        prompt_template=PROMPT_TEMPLATE,
+    )
 )
-class GeneralQAAdapter(DataAdapter):
+class GeneralQAAdapter(DefaultDataAdapter):
 
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
 
-    def load(self, dataset_name_or_path: str = None, subset_list: list = None, **kwargs) -> dict:
-        """
-        Load dataset from the given path or dataset name.
-
-        Args:
-            dataset_name_or_path (str): Path to dataset directory or file.
-            subset_list (list): List of subset names to load.
-
-        Returns:
-            dict: Loaded dataset organized by subset.
-        """
-        dataset_name_or_path = dataset_name_or_path or self.dataset_id
-        subset_list = subset_list or self.subset_list
-
-        data_file_dict = defaultdict(str)
-        data_item_dict = defaultdict(list)
-
-        # get data file path and subset name
-        if os.path.isdir(dataset_name_or_path):
-            for subset_name in subset_list:
-                data_file_dict[subset_name] = os.path.join(dataset_name_or_path, f'{subset_name}.jsonl')
-        elif os.path.isfile(dataset_name_or_path):
-            cur_subset_name = os.path.splitext(os.path.basename(dataset_name_or_path))[0]
-            data_file_dict[cur_subset_name] = dataset_name_or_path
-        else:
-            raise ValueError(f'Invalid dataset path: {dataset_name_or_path}')
-
-        # load data from local disk
-        try:
-            for subset_name, file_path in data_file_dict.items():
-                data_item_dict[subset_name] = jsonl_to_list(file_path)
-        except Exception as e:
-            raise ValueError(f'Failed to load data from {self.dataset_id}, got error: {e}')
-
-        data_dict = {subset_name: {'test': data_item_dict[subset_name]} for subset_name in data_file_dict.keys()}
-
-        return data_dict
-
-    def gen_prompt(self, input_d: dict, subset_name: str, few_shot_list: list, **kwargs) -> dict:
-        """
-        Generate prompt for the model based on input data.
-
-        Args:
-            input_d (dict): Input data dictionary.
-            subset_name (str): Name of the subset.
-            few_shot_list (list): List of few-shot examples.
-
-        Returns:
-            dict: Dictionary containing the generated prompt.
-        """
-        messages = input_d.get('messages')
-        query = input_d.get('question', '') or input_d.get('query', '')
-        system_prompt = input_d.get('system')
-        prompt = self.prompt_template.format(query=query)
-        return self.gen_prompt_data(prompt, system_prompt=system_prompt, messages=messages)
-
-    def get_gold_answer(self, input_d: dict) -> str:
-        """
-        Extract the gold (reference) answer from the input data.
-
-        Args:
-            input_d (dict): Input data dictionary.
-
-        Returns:
-            str: Gold answer string.
-        """
-        return input_d.get('answer') or input_d.get('response')
-
-    def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = 'checkpoint') -> str:
-        """
-        Parse the prediction result.
-
-        Args:
-            result (str): Model prediction result.
-            raw_input_d (dict, optional): Original input data.
-            eval_type (str): Evaluation type.
+    def load_from_disk(self, **kwargs):
+        return super().load_from_disk(use_local_loader=True)
 
-        Returns:
-            str: Parsed prediction result.
-        """
-        return result
-
-    def match(self, gold: str, pred: str) -> dict:
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
         """
-        Compute metric scores between gold and predicted answers.
+        Convert a data record to a Sample object.
 
         Args:
-            gold (str): Gold answer.
-            pred (str): Predicted answer.
+            record (Dict[str, Any]): Input data record.
 
         Returns:
-            dict: Dictionary of computed metric scores.
+            Sample: Sample object with input, target, and metadata.
         """
-        # reference free metrics
-        if gold is None:
-            return {'AverageAccuracy': -1}
-
-        # calculate rouge and bleu scores
-        res = dict()
-        if 'AverageRouge' in self.metric_list:
-            from evalscope.metrics.rouge_metric import compute_rouge_score_one_sample_zh
-
-            rouge_dict = compute_rouge_score_one_sample_zh([pred], [gold])
-            res.update(rouge_dict)
-        if 'AverageBLEU' in self.metric_list:
-            from evalscope.metrics import bleu_ngram_one_sample
+        query = record.get('question') or record.get('query')
+        answer = record.get('answer') or record.get('response')
+        system_prompt = record.get('system')
+        messages = record.get('messages')
+
+        message_list = []
+        if messages:
+            message_list = [dict_to_chat_message(m) for m in messages]
+        else:
+            if system_prompt:
+                message_list.append(ChatMessageSystem(content=system_prompt))
+            message_list.append(ChatMessageUser(content=query))
 
-            bleu_dict = bleu_ngram_one_sample(pred, gold)
-            res.update(bleu_dict)
-        return res
+        return Sample(input=message_list, target=answer or '')
 
-    def compute_metric(self, review_res_list: Union[List[dict], List[List[dict]]], **kwargs) -> List[dict]:
+    def match_score(
+        self, original_prediction: str, filtered_prediction: str, reference: str, task_state: TaskState
+    ) -> Score:
         """
-        Compute weighted mean of the metric scores for all samples.
-
-        Args:
-            review_res_list (list): List of metric score dictionaries.
-
-        Returns:
-            list: List of dictionaries with averaged metric results.
+        Calculate evaluation scores by comparing prediction with reference.
         """
-        items = super().compute_dict_metric(review_res_list, **kwargs)
-        return [{'metric_name': k, 'score': mean(v), 'num': len(v)} for k, v in items.items()]
+        # Initialize the score object with prediction details
+        score = Score(
+            extracted_prediction=filtered_prediction,
+            prediction=original_prediction,
+        )
+
+        # Calculate scores for each configured metric
+        for metric in self.metric_list:
+            try:
+                if metric == 'Rouge':
+                    from evalscope.metrics.rouge_metric import compute_rouge_score_one_sample_zh
+
+                    score.value.update(compute_rouge_score_one_sample_zh([filtered_prediction], [reference]))
+                elif metric == 'BLEU':
+                    from evalscope.metrics import bleu_ngram_one_sample
+
+                    score.value.update(bleu_ngram_one_sample(filtered_prediction, reference))
+            except Exception as e:
+                logger.error(f'Error calculating metric {metric}: {e}')
+                return None
+
+        score.main_score_name = 'Rouge-L-R'
+        return score

evalscope/benchmarks/gpqa/gpqa_adapter.py

@@ -1,63 +1,70 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
 import os
 import random
 import re
-
-from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.constants import EvalType, OutputType
-from evalscope.metrics import exact_match
-
-
-@Benchmark.register(
-    name='gpqa',
-    pretty_name='GPQA',
-    tags=['MCQ', 'Knowledge'],
-    description=
-    'GPQA is a dataset for evaluating the reasoning ability of large language models (LLMs) on complex mathematical problems. It contains questions that require step-by-step reasoning to arrive at the correct answer.',  # noqa: E501
-    dataset_id='modelscope/gpqa',
-    model_adapter=OutputType.GENERATION,
-    output_types=[OutputType.MULTIPLE_CHOICE, OutputType.GENERATION],
-    subset_list=['gpqa_extended', 'gpqa_main', 'gpqa_diamond'],
-    metric_list=['AveragePass@1'],
-    few_shot_num=5,
-    train_split=None,
-    eval_split='train',  # only have train split
-    prompt_template='{query}\nPlease reason step by step, and put your final answer within \\boxed{{}}.',
+from typing import Any, Dict
+
+from evalscope.api.benchmark import BenchmarkMeta, MultiChoiceAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.logger import get_logger
+from evalscope.utils.multi_choices import FEW_SHOT_TEMPLATE, MultipleChoiceTemplate
+
+logger = get_logger()
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='gpqa_diamond',
+        pretty_name='GPQA-Diamond',
+        tags=[Tags.KNOWLEDGE, Tags.MULTIPLE_CHOICE],
+        description=
+        'GPQA is a dataset for evaluating the reasoning ability of large language models (LLMs) on complex mathematical problems. It contains questions that require step-by-step reasoning to arrive at the correct answer.',  # noqa: E501
+        dataset_id='AI-ModelScope/gpqa_diamond',
+        metric_list=['acc'],
+        few_shot_num=0,
+        train_split=None,
+        eval_split='train',  # only have train split
+        prompt_template=MultipleChoiceTemplate.SINGLE_ANSWER_COT,
+    )
 )
-class GPQAAdapter(DataAdapter):
+class GPQAAdapter(MultiChoiceAdapter):
 
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
 
-        self.choices = ['A', 'B', 'C', 'D']
-        if self.few_shot_num and self.few_shot_num > 0:
-            self.prompt_prefix = 'Here are some example questions from experts. Answer the final question yourself, following the format of the previous questions exactly.\n'  # noqa: E501
-            self.prompt_prefix += open(
-                os.path.join(os.path.dirname(__file__), 'chain_of_thought.txt'), 'r',
-                encoding='utf-8').read() + '\nQuestion: '
-        else:
-            self.prompt_prefix = 'What is the correct answer to this question:'
-
-    def gen_prompt(self, input_d: dict, subset_name: str, few_shot_list: list, **kwargs) -> dict:
-        """
-        Generate model prompt from input data.
-        example:
-        {
-            "question":"Two people are playing the following game. A fair coin is tossed into the air. Person A says that in a single toss of the coin, the tail will come. So it's like the first shot or the third shot or the fifth shot. Person B says that the coin will come with a double toss. So like the second, fourth, sixth or eighth shot. Imagine this game played forever. What is the probability that person A wins this game?",
-            "choice1":"1/2",
-            "choice2":"1/4",
-            "choice3":"2/3",
-            "choice4":"1/8",
-            "answer":"C",
-        }
-        """  # noqa: E501
-        processed_input_d = self.__process_input(input_d)
-        input_d['answer'] = processed_input_d['answer']  # add answer to input_d for answer extraction
-        query = self.prompt_prefix + f"{input_d['Question']}\n{self.__form_options(processed_input_d['choices'])}"  # noqa: E501
-
-        prompt = self.prompt_template.format(query=query)
-        return self.gen_prompt_data(prompt)
-
-    def __process_input(self, input_d: dict) -> dict:
+        if self.few_shot_num > 0 and self.few_shot_num != 5:
+            logger.warning(
+                f'Only support few_shot_num 0 or 5 for {self.dataset_id}, but got {self.few_shot_num}. Use 5-shot by default.'  # noqa: E501
+            )
+            self.few_shot_num = 5
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        # Process the input to create shuffled choices and correct answer
+        processed_data = self._process_input(record)
+
+        return Sample(
+            input=record['Question'],
+            choices=processed_data['choices'],
+            target=processed_data['answer'],
+            subset_key=record.get('subset', ''),
+            metadata={
+                'correct_answer':
+                record['Correct Answer'],
+                'incorrect_answers':
+                [record['Incorrect Answer 1'], record['Incorrect Answer 2'], record['Incorrect Answer 3']],
+            },
+        )
+
+    def format_fewshot_template(self, fewshot, sample):
+        from .prompt import FEW_SHOT_SAMPLES
+
+        return FEW_SHOT_TEMPLATE.format(fewshot=FEW_SHOT_SAMPLES, ) + self.format_prompt_template(sample)
+
+    def _process_input(self, input_d: dict) -> dict:
+        """Process input to shuffle choices and determine correct answer letter."""
 
         def preprocess(text):
             if text is None:
@@ -77,53 +84,7 @@ class GPQAAdapter(DataAdapter):
         random.shuffle(choices)
         correct_answer_index = choices.index(preprocess(input_d['Correct Answer']))
 
-        out_doc = {
-            'choices': [choices[0], choices[1], choices[2], choices[3]],
+        return {
+            'choices': choices,
             'answer': f'{chr(65 + correct_answer_index)}',
         }
-        return out_doc
-
-    def __form_options(self, options: list):
-        option_str = 'Choices:\n'
-        for opt, choice in zip(options, self.choices):
-            option_str += f'({choice}) {opt}' + '\n'
-        return option_str
-
-    def get_gold_answer(self, input_d: dict) -> str:
-        """
-        Parse the raw input labels (gold).
-        """
-        return input_d['answer']
-
-    def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = EvalType.CHECKPOINT) -> str:
-        """
-        Parse the predicted result and extract proper answer.
-        """
-        if self.model_adapter == OutputType.MULTIPLE_CHOICE:
-            return result
-        else:
-            return GPQAAdapter.get_multiple_choice_answer(result)
-
-    def match(self, gold: str, pred: str) -> float:
-        """
-        Match the gold answer and the predicted answer.
-        """
-        return exact_match(gold=gold, pred=pred)
-
-    @staticmethod
-    def get_multiple_choice_answer(pred: str):
-        tmp = re.findall(r'\b(A|B|C|D)\b', pred.upper())
-        if tmp:
-            pred = tmp
-        else:
-            pred = [pred.strip().strip('.')]
-
-        if len(pred) == 0:
-            pred = ''
-        else:
-            pred = pred[-1]
-
-        # Remove the period at the end, again!
-        pred = pred.rstrip('.').rstrip('/')
-
-        return pred