evalscope 0.17.1__py3-none-any.whl → 1.0.1__py3-none-any.whl

evalscope/benchmarks/frames/frames_adapter.py

@@ -1,6 +1,15 @@
-from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.constants import EvalType, OutputType
-from evalscope.metrics import LLMJudge, exact_match
+import os
+from typing import Any, Dict
+
+from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter
+from evalscope.api.dataset import DatasetDict, LocalDataLoader, Sample
+from evalscope.api.evaluator import TaskState
+from evalscope.api.metric import Score
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
 
 TEMPLATE_0SHOT = """Please read the following text and answer the question below.
 
@@ -13,52 +22,83 @@ TEMPLATE_0SHOT = """Please read the following text and answer the question below
 Format your response as follows: "Therefore, the answer is (insert answer here)"."""
 
 
-@Benchmark.register(
-    name='frames',
-    pretty_name='FRAMES',
-    tags=['Reasoning', 'Long Context'],
-    description=
-    'FRAMES is a comprehensive evaluation dataset designed to test the capabilities of Retrieval-Augmented Generation (RAG) systems across factuality, retrieval accuracy, and reasoning.',  # noqa: E501
-    dataset_id='iic/frames',
-    model_adapter=OutputType.GENERATION,
-    output_types=[OutputType.GENERATION],
-    metric_list=['AverageAccuracy'],
-    few_shot_num=0,
-    train_split=None,
-    eval_split='test',
-    prompt_template=TEMPLATE_0SHOT,
+@register_benchmark(
+    BenchmarkMeta(
+        name='frames',
+        pretty_name='FRAMES',
+        tags=[Tags.REASONING, Tags.LONG_CONTEXT],
+        description=
+        'FRAMES is a comprehensive evaluation dataset designed to test the capabilities of Retrieval-Augmented Generation (RAG) systems across factuality, retrieval accuracy, and reasoning.',  # noqa: E501
+        dataset_id='iic/frames',
+        metric_list=['acc'],
+        eval_split='test',
+        prompt_template=TEMPLATE_0SHOT,
+    )
 )
-class FramesAdapter(DataAdapter):
+class FramesAdapter(DefaultDataAdapter):
 
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
+        self._use_llm_judge = True  # Enable LLM judge for FRAMES
+
+    def load(self):
+        # Try to load dataset from local disk
+        dataset_name_or_path = self.dataset_id
+        if os.path.exists(dataset_name_or_path):
+            logger.info(f'Loading dataset from {dataset_name_or_path}')
+            dataset_path = dataset_name_or_path
+        else:
+            from modelscope import dataset_snapshot_download
 
-    def load(self, **kwargs):
-        # default load with snapshot
-        kwargs['file_structure'] = {'default': ['test.jsonl']}
-        data_dict = super().load_with_snapshot(**kwargs)
-        return data_dict
+            # Load dataset from remote
+            logger.info(f'Loading dataset from modelscope: > dataset_name: {dataset_name_or_path}')
+            # download dataset snapshot
+            dataset_path = dataset_snapshot_download(dataset_name_or_path, allow_file_pattern='test.jsonl')
 
-    def gen_prompt(self, input_d: dict, subset_name: str, few_shot_list: list, **kwargs) -> dict:
-        """
-        Generate model prompt from input data.
-        """
-        context = '\n'.join([f"{i['title']}\n{i['text']}" for i in input_d['wiki_items']])
-        question = input_d['Prompt']
-        prompt = self.prompt_template.format(context=context, question=question)
-        return self.gen_prompt_data(prompt)
+        dataset = LocalDataLoader(
+            data_id_or_path=dataset_path,
+            split=self.eval_split,
+            sample_fields=self.record_to_sample,
+            subset='test',
+            limit=self.limit,
+            repeats=self.repeats,
+            shuffle=self.shuffle,
+        ).load()
 
-    def get_gold_answer(self, input_d: dict) -> str:
-        """
-        Parse the raw input labels (gold).
+        test_dataset = DatasetDict({'test': dataset})
+
+        return test_dataset, None
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
         """
-        return input_d['Answer']
+        Convert a data record to a Sample object.
 
-    def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = EvalType.CHECKPOINT) -> str:
+        Args:
+            record (Dict[str, Any]): Input data record.
+
+        Returns:
+            Sample: Sample object with input, target, and metadata.
+        """
+        context = '\n'.join([f"{i['title']}\n{i['text']}" for i in record['wiki_items']])
+        question = record['Prompt']
+
+        return Sample(
+            input=question, target=record['Answer'], metadata={
+                'context': context,
+                'wiki_items': record['wiki_items']
+            }
+        )
+
+    def format_prompt_template(self, sample):
+        context = sample.metadata['context']
+        question = sample.input
+        return self.prompt_template.format(context=context, question=question)
+
+    def extract_answer(self, prediction: str, task_state: TaskState):
         """
-        Parse the predicted result and extract proper answer.
+        Extract the answer from the model prediction.
         """
-        response = result.replace('*', '')
+        response = prediction.replace('*', '')
 
         if 'the answer is' in response:
             ans = response.rsplit('the answer is', 1)[-1].strip().strip('.').strip()
@@ -67,25 +107,69 @@ class FramesAdapter(DataAdapter):
 
         return ans
 
-    def match(self, gold: str, pred: str) -> float:
+    def match_score(
+        self,
+        original_prediction: str,
+        filtered_prediction: str,
+        reference: str,
+        task_state: TaskState,
+    ) -> Score:
         """
-        Match the gold answer and the predicted answer.
+        Calculate accuracy score by matching prediction with reference.
         """
+        from evalscope.metrics import exact_match
         from .utils import normalize_answer
-        gold = normalize_answer(gold)
-        pred = normalize_answer(pred)
-        return exact_match(gold=gold, pred=pred)
 
-    def llm_match(self, gold: str, pred: str, judge: LLMJudge, **kwargs) -> float:
+        score = Score(
+            extracted_prediction=filtered_prediction,
+            prediction=original_prediction,
+        )
+
+        gold = normalize_answer(reference)
+        pred = normalize_answer(filtered_prediction)
+        accuracy = exact_match(gold=gold, pred=pred)
+
+        score.value = {'acc': accuracy}
+        score.main_score_name = 'acc'
+
+        return score
+
+    def llm_match_score(
+        self,
+        original_prediction: str,
+        filtered_prediction: str,
+        reference: str,
+        task_state: TaskState,
+    ) -> Score:
+        """
+        Use LLM judge to evaluate the prediction against the reference.
+        """
         from .utils import GENERAL_ORM_PROMPT, ORM_USER_TEMPLATE
 
-        raw_input = kwargs.get('raw_input', None)
-        question = raw_input['Prompt']
-        # get grading response
-        prompt = ORM_USER_TEMPLATE.format(problem=question, answer_1=gold, answer_2=pred)
-        orm_response = judge(prompt=prompt, system_prompt=GENERAL_ORM_PROMPT)
-        # parse grading response
+        score = Score(
+            extracted_prediction=filtered_prediction,
+            prediction=original_prediction,
+        )
+
+        question = task_state.input_text
+
+        # Get grading response
+        prompt = ORM_USER_TEMPLATE.format(problem=question, answer_1=reference, answer_2=filtered_prediction)
+        orm_response = self.llm_judge.judge(prompt, system_prompt=GENERAL_ORM_PROMPT)
+
+        # Parse grading response
         if 'YES' in orm_response:
-            return 1.0
+            accuracy = 1.0
         else:
-            return 0.0
+            accuracy = 0.0
+
+        score.value = {'acc': accuracy}
+        score.explanation = f'LLM judge: {orm_response}'
+        score.metadata = {
+            'source': 'llm_judge',
+            'judge_strategy': self.judge_strategy,
+            'model': self.llm_judge.model_id
+        }
+        score.main_score_name = 'acc'
+
+        return score

evalscope/benchmarks/general_arena/general_arena_adapter.py

@@ -1,16 +1,19 @@
+# flake8: noqa: E501
 import glob
 import os
 from collections import defaultdict
-from typing import Any, List
-
-from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.constants import EvalType
-from evalscope.metrics import Metric, mean, metric_registry
+from typing import Any, Dict, List
+
+from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter
+from evalscope.api.dataset import DatasetDict, DictDataLoader, Sample
+from evalscope.api.evaluator import TaskState
+from evalscope.api.messages.chat_message import ChatMessageUser
+from evalscope.api.metric import AggScore, SampleScore, Score
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
 from evalscope.report import Report, ReportKey
 from evalscope.utils.logger import get_logger
 
-# flake8: noqa
-
 logger = get_logger()
 
 GRADER_SYSTEM_PROMPT = "Please act as an impartial judge and evaluate the quality of the responses provided by two AI assistants to the user prompt displayed below. You will be given assistant A's answer and assistant B's answer. Your job is to evaluate which assistant's answer is better.\n\nBegin your evaluation by generating your own answer to the prompt. You must provide your answers before judging any answers.\n\nWhen evaluating the assistants' answers, compare both assistants' answers with your answer. You must identify and correct any mistakes or inaccurate information.\n\nThen consider if the assistant's answers are helpful, relevant, and concise. Helpful means the answer correctly responds to the prompt or follows the instructions. Note when user prompt has any ambiguity or more than one interpretation, it is more helpful and appropriate to ask for clarifications or more information from the user than providing an answer based on assumptions. Relevant means all parts of the response closely connect or are appropriate to what is being asked. Concise means the response is clear and not verbose or excessive.\n\nThen consider the creativity and novelty of the assistant's answers when needed. Finally, identify any missing important information in the assistants' answers that would be beneficial to include when responding to the user prompt.\n\nAfter providing your explanation, you must output only one of the following choices as your final verdict with a label:\n\n1. Assistant A is significantly better: [[A>>B]]\n2. Assistant A is slightly better: [[A>B]]\n3. Tie, relatively the same: [[A=B]]\n4. Assistant B is slightly better: [[B>A]]\n5. Assistant B is significantly better: [[B>>A]]\n\nExample output: \"My final verdict is tie: [[A=B]]\"."  # noqa: E501
@@ -19,59 +22,81 @@ GRADER_TEMPLATE = "<|User Prompt|>\n{question}\n\n<|The Start of Assistant A's A
 )  # noqa: E501
 
 
-@Benchmark.register(
-    name='general_arena',
-    pretty_name='GeneralArena',
-    tags=['Custom', 'Arena'],
-    description=
-    'GeneralArena is a custom benchmark designed to evaluate the performance of large language models in a competitive setting, '
-    'where models are pitted against each other in custom tasks to determine their relative strengths and weaknesses. You should '
-    'provide the model outputs in the format of a list of dictionaries, where each dictionary contains the model name and its report path. '
-    'For detailed instructions on how to use this benchmark, please refer to the [Arena User Guide](https://evalscope.readthedocs.io/zh-cn/latest/user_guides/arena.html).',
-    dataset_id='general_arena',
-    metric_list=['winrate'],
-    few_shot_num=0,
-    train_split=None,
-    eval_split='test',
-    system_prompt=GRADER_SYSTEM_PROMPT,
-    prompt_template=GRADER_TEMPLATE,
-    extra_params={
-        'models': [{
-            'name': 'qwen-plus',
-            'report_path': 'outputs/20250627_172550/reports/qwen-plus'
-        }, {
-            'name': 'qwen2.5-7b',
-            'report_path': 'outputs/20250627_172817/reports/qwen2.5-7b-instruct'
-        }],
-        'baseline':
-        'qwen2.5-7b'
-    })
-class GeneralArenaAdapter(DataAdapter):
+@register_benchmark(
+    BenchmarkMeta(
+        name='general_arena',
+        pretty_name='GeneralArena',
+        tags=[Tags.CUSTOM, Tags.ARENA],
+        description=
+        'GeneralArena is a custom benchmark designed to evaluate the performance of large language models in a competitive setting, '
+        'where models are pitted against each other in custom tasks to determine their relative strengths and weaknesses. You should '
+        'provide the model outputs in the format of a list of dictionaries, where each dictionary contains the model name and its report path. '
+        'For detailed instructions on how to use this benchmark, please refer to the [Arena User Guide](https://evalscope.readthedocs.io/zh-cn/latest/user_guides/arena.html).',
+        dataset_id='general_arena',
+        metric_list=['winrate'],
+        few_shot_num=0,
+        train_split=None,
+        eval_split='test',
+        system_prompt=GRADER_SYSTEM_PROMPT,
+        prompt_template=GRADER_TEMPLATE,
+        extra_params={
+            'models': [{
+                'name': 'qwen-plus',
+                'report_path': 'outputs/20250627_172550/reports/qwen-plus'
+            }, {
+                'name': 'qwen2.5-7b',
+                'report_path': 'outputs/20250627_172817/reports/qwen2.5-7b-instruct'
+            }],
+            'baseline':
+            'qwen2.5-7b'
+        }
+    )
+)
+class GeneralArenaAdapter(DefaultDataAdapter):
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
 
-        # register metrics
-        metric_registry.register(Metric(name='winrate', object=mean))
+        self._use_llm_judge = True
 
-        # whether to use LLM as a judge
-        self.llm_as_a_judge = True
+        self.models = self.extra_params.get('models', [])
+        self.baseline = self.extra_params.get('baseline', None)
 
-        extra_params = kwargs.get('extra_params', {})
-        self.models = extra_params.get('models', [])
-        self.baseline = extra_params.get('baseline', None)
-
-    def load(self, **kwargs):
+    def load(self):
+        """Load dataset by processing model reports."""
         self._check_names()
         self._check_reports()
         self._check_datasets()
         logger.info(f'Overall datasets: {self.overall_datasets}')
         dataset_model_dict = self._load_common_datasets()
-        data_dict = self._build_pair_wise_data(dataset_model_dict)
-        return data_dict
-
-    def gen_prompt(self, input_d, subset_name, few_shot_list, **kwargs):
-        return self.gen_prompt_data(input_d['question'])
+        datasets = self._build_pair_wise_data(dataset_model_dict)
+
+        # Convert to DatasetDict format
+        dataset_dict = {}
+        for subset_name, samples in datasets.items():
+            dataset = DictDataLoader(
+                dict_list=samples,
+                limit=self.limit,
+                shuffle=self.shuffle,
+                repeats=self.repeats,
+                sample_fields=self.record_to_sample
+            ).load()
+            dataset_dict[subset_name] = dataset
+
+        test_dataset = DatasetDict(dataset_dict)
+        return test_dataset, None
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        """Convert a data record to a Sample object."""
+        return Sample(
+            input=[ChatMessageUser(content=record['question'])],
+            target=record['answer_2'],  # baseline answer
+            metadata={
+                'answer_1': record['answer_1'],
+                'model_1': record['model_1'],
+                'model_2': record['model_2'],
+            }
+        )
 
     def _check_names(self):
         """Check the names of the models and baseline."""
@@ -119,7 +144,8 @@ class GeneralArenaAdapter(DataAdapter):
 
     def _load_common_datasets(self):
         """Load common datasets from the local path."""
-        from evalscope.utils import OutputsStructure, jsonl_to_list
+        from evalscope.utils import OutputsStructure
+        from evalscope.utils.io_utils import jsonl_to_list
 
         dataset_dict = defaultdict(dict)
         for dataset_name, subset_name in self.overall_datasets:
@@ -128,7 +154,8 @@ class GeneralArenaAdapter(DataAdapter):
                 dataset_file_path = os.path.join(dataset_path, f'{dataset_name}_{subset_name}.jsonl')
                 if not os.path.exists(dataset_file_path):
                     raise ValueError(
-                        f'Dataset {dataset_name} with subset {subset_name} not found in model {model["name"]}.')
+                        f'Dataset {dataset_name} with subset {subset_name} not found in model {model["name"]}.'
+                    )
                 dataset = jsonl_to_list(dataset_file_path)
                 # sort by index
                 dataset.sort(key=lambda x: x.get('index'))
@@ -138,9 +165,10 @@ class GeneralArenaAdapter(DataAdapter):
 
     def _build_pair_wise_data(self, dataset_dict):
         """Build pairwise data for the models."""
+        from evalscope.api.evaluator import ReviewResult
         from .utils import process_review_item
 
-        pairwise_data = defaultdict(dict)
+        pairwise_data = defaultdict(list)
         for (dataset_name, subset_name), model_data in dataset_dict.items():
             if len(model_data) < 2:
                 logger.warning(f'Not enough models for dataset {dataset_name} with subset {subset_name}. Skipping.')
@@ -152,8 +180,13 @@ class GeneralArenaAdapter(DataAdapter):
                     continue
                 pairs = []
                 for model_item, baseline_item in zip(model_data[name], model_data[self.baseline]):
+                    # Convert to ReviewResult objects like in get_model_prediction
+                    model_review = ReviewResult.model_validate(model_item)
+                    baseline_review = ReviewResult.model_validate(baseline_item)
+
                     for model_choice, baseline_choice in zip(
-                            process_review_item(model_item), process_review_item(baseline_item)):
+                        process_review_item(model_review), process_review_item(baseline_review)
+                    ):
                         pairs.append({
                             'question': model_choice['Question'],
                             'answer_1': model_choice['Generated'],
@@ -161,23 +194,26 @@ class GeneralArenaAdapter(DataAdapter):
                             'model_1': name,
                             'model_2': self.baseline
                         })
-                pairwise_data[f'{dataset_name}&{subset_name}@{name}&{self.baseline}'][self.eval_split] = pairs
+                pairwise_data[f'{dataset_name}&{subset_name}@{name}&{self.baseline}'] = pairs
 
         return pairwise_data
 
-    def llm_match(self, gold, pred, judge=None, **kwargs):
+    def llm_match_score(
+        self, original_prediction: str, filtered_prediction: str, reference: str, task_state: TaskState
+    ) -> Score:
+        """Use LLM as a judge to evaluate the predicted answer against the baseline."""
         from .utils import get_judge_score, post_process_result
 
-        try:
-            raw_input = kwargs.get('raw_input', None)
-            question = raw_input['question']
-            answer_1 = raw_input['answer_1']
-            answer_2 = raw_input['answer_2']
-            model_1 = raw_input['model_1']
-            model_2 = raw_input['model_2']
-        except KeyError as e:
-            logger.error(f'Missing key in raw input: {e}. Raw input: {raw_input}')
-            raise
+        score = Score(
+            extracted_prediction=filtered_prediction,
+            prediction=original_prediction,
+        )
+
+        question = task_state.input_text
+        answer_1 = task_state.metadata['answer_1']
+        answer_2 = reference  # baseline answer
+        model_1 = task_state.metadata['model_1']
+        model_2 = task_state.metadata['model_2']
 
         system_template = self.system_prompt
         prompt_template = self.prompt_template
@@ -185,9 +221,11 @@ class GeneralArenaAdapter(DataAdapter):
         prompt1 = prompt_template.format(question=question, answer_1=answer_1, answer_2=answer_2)
         # reverse the order
         prompt2 = prompt_template.format(question=question, answer_1=answer_2, answer_2=answer_1)
+
         # get grading response
-        game1_response = judge(prompt1, system_prompt=system_template)
-        game2_response = judge(prompt2, system_prompt=system_template)
+        game1_response = self.llm_judge.judge(prompt1, system_prompt=system_template)
+        game2_response = self.llm_judge.judge(prompt2, system_prompt=system_template)
+
         # parse grading response
         # game1
         res1 = post_process_result(game1_response)
@@ -195,9 +233,9 @@ class GeneralArenaAdapter(DataAdapter):
         # game2
         res2 = post_process_result(game2_response)
         score2 = get_judge_score(res2, reverse=True)
-        return {
-            'score':
-            mean([score1, score2]),
+
+        battle_result = {
+            'score': (score1 + score2) / 2,
             'games': [
                 {
                     'model_a': model_1,
@@ -214,31 +252,38 @@ class GeneralArenaAdapter(DataAdapter):
             ]
         }
 
-    def compute_metric(self, review_res_list: List[dict], **kwargs) -> List[dict]:
-        """
-        compute score of the model
-        """
+        score.value = {'score': battle_result['score']}
+        score.explanation = f'LLM judge battles: Game1: {game1_response[:100]}... Game2: {game2_response[:100]}...'
+        score.metadata = {
+            'source': 'llm_judge',
+            'judge_strategy': getattr(self, 'judge_strategy', 'default'),
+            'model': self.llm_judge.model_id if hasattr(self.llm_judge, 'model_id') else 'unknown',
+            'battle_result': battle_result
+        }
+        score.main_score_name = 'score'
+
+        return score
+
+    def aggregate_scores(self, sample_scores: List[SampleScore]) -> List[AggScore]:
+        """Aggregate scores to compute winrate."""
         import numpy as np
         import pandas as pd
 
         from .utils import compute_mle_elo, get_battles_from_row, get_bootstrap_result, get_win_rate_column
 
-        if isinstance(review_res_list[0], list):
-            review_res_list = [item for sublist in review_res_list for item in sublist]
-
-        battles = pd.concat([get_battles_from_row(res) for res in review_res_list])
+        battles = pd.concat([get_battles_from_row(res.score.metadata['battle_result']) for res in sample_scores])
 
         bt_model_coef = compute_mle_elo(battles, baseline_model=self.baseline)
 
         bootstrap_model_coef = get_bootstrap_result(
-            battles, func_compute_elo=compute_mle_elo, num_round=100, baseline_model=self.baseline)
+            battles, func_compute_elo=compute_mle_elo, num_round=100, baseline_model=self.baseline
+        )
 
         stats = pd.DataFrame()
         stats['results'] = None
         stats['results'] = stats['results'].astype('object')
 
         for i, model in enumerate(bt_model_coef.index):
-            # assert model in bootstrap_elo_lu.columns
             stats.at[i, 'model'] = model
             stats.at[i, 'score'] = bt_model_coef[model]
             stats.at[i, 'lower'] = np.percentile(bootstrap_model_coef[model], 2.5)
@@ -249,20 +294,25 @@ class GeneralArenaAdapter(DataAdapter):
         metrics_dict['winrate_lower'] = get_win_rate_column(stats, 'lower', self.baseline).to_dict()
         metrics_dict['winrate_upper'] = get_win_rate_column(stats, 'upper', self.baseline).to_dict()
 
-        metrics = []
+        agg_scores = []
         for metric_name, models in metrics_dict.items():
-            for model_name, score in models.items():
+            for model_name, score_val in models.items():
                 if model_name == self.baseline:
                     continue
-                metrics.append({'metric_name': metric_name, 'score': score, 'num': len(review_res_list)})
-        return metrics
+                agg_scores.append(AggScore(score=score_val, metric_name=metric_name, num=len(sample_scores)))
+
+        return agg_scores
 
-    def post_process_report(self, report: 'Report', **kwargs):
+    def extract_answer(self, prediction, task_state):
+        # NOTE: This is a hacky way to extract the answer from the prediction
+        return task_state.metadata['answer_1']
+
+    def _on_generate_report_end(self, report: 'Report', output_dir: str, **kwargs):
         """Post-process the report to convert it to a DataFrame with winrate leaderboards."""
         import pandas as pd
         import tabulate
 
-        report_path = kwargs.get('report_path')
+        report_path = output_dir
         leaderboard_file = os.path.join(report_path, 'leaderboard.txt')
 
         # Ensure report directory exists
@@ -288,7 +338,8 @@ class GeneralArenaAdapter(DataAdapter):
             """Format DataFrame as leaderboard with CI."""
             # Pivot to get winrate, winrate_lower, winrate_upper as columns
             pivot_df = data_df.pivot_table(
-                index=[ReportKey.model_name], columns=ReportKey.metric_name, values=ReportKey.score, aggfunc='first')
+                index=[ReportKey.model_name], columns=ReportKey.metric_name, values=ReportKey.score, aggfunc='first'
+            )
 
             # Add baseline model with 50% winrate
             baseline_data = {'winrate': 0.5, 'winrate_lower': 0.5, 'winrate_upper': 0.5}
@@ -392,20 +443,11 @@ class GeneralArenaAdapter(DataAdapter):
             subset_df = parsed_df[(parsed_df['dataset_name'] == dataset_name)
                                   & (parsed_df['subset_name'] == subset_name)]
             leaderboard_outputs.append(
-                format_leaderboard(subset_df, f'=== SUBSET LEADERBOARD: {dataset_name} - {subset_name} ==='))
+                format_leaderboard(subset_df, f'=== SUBSET LEADERBOARD: {dataset_name} - {subset_name} ===')
+            )
 
         # Write all leaderboard outputs to file
         with open(leaderboard_file, 'w', encoding='utf-8') as f:
             f.write('\n'.join(leaderboard_outputs))
 
         logger.info(f'Leaderboard results saved to: {leaderboard_file}')
-
-    def get_gold_answer(self, input_d):
-        return f"model_1: {input_d['model_1']}\n---\n" + input_d['answer_1']
-
-    def llm_parse_pred_result(self, result, raw_input_d=None, eval_type=EvalType.CHECKPOINT):
-        return f"model_2: {raw_input_d['model_2']}\n---\n" + raw_input_d['answer_2']
-
-    def match(self, gold, pred):
-        logger.warning(f'Please use LLMJudge to match the result for {self.name}')
-        return