evalscope 0.17.1__py3-none-any.whl → 1.0.0__py3-none-any.whl

evalscope/benchmarks/general_arena/general_arena_adapter.py

@@ -1,16 +1,19 @@
+# flake8: noqa: E501
 import glob
 import os
 from collections import defaultdict
-from typing import Any, List
-
-from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.constants import EvalType
-from evalscope.metrics import Metric, mean, metric_registry
+from typing import Any, Dict, List
+
+from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter
+from evalscope.api.dataset import DatasetDict, DictDataLoader, Sample
+from evalscope.api.evaluator import TaskState
+from evalscope.api.messages.chat_message import ChatMessageUser
+from evalscope.api.metric import AggScore, SampleScore, Score
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
 from evalscope.report import Report, ReportKey
 from evalscope.utils.logger import get_logger
 
-# flake8: noqa
-
 logger = get_logger()
 
 GRADER_SYSTEM_PROMPT = "Please act as an impartial judge and evaluate the quality of the responses provided by two AI assistants to the user prompt displayed below. You will be given assistant A's answer and assistant B's answer. Your job is to evaluate which assistant's answer is better.\n\nBegin your evaluation by generating your own answer to the prompt. You must provide your answers before judging any answers.\n\nWhen evaluating the assistants' answers, compare both assistants' answers with your answer. You must identify and correct any mistakes or inaccurate information.\n\nThen consider if the assistant's answers are helpful, relevant, and concise. Helpful means the answer correctly responds to the prompt or follows the instructions. Note when user prompt has any ambiguity or more than one interpretation, it is more helpful and appropriate to ask for clarifications or more information from the user than providing an answer based on assumptions. Relevant means all parts of the response closely connect or are appropriate to what is being asked. Concise means the response is clear and not verbose or excessive.\n\nThen consider the creativity and novelty of the assistant's answers when needed. Finally, identify any missing important information in the assistants' answers that would be beneficial to include when responding to the user prompt.\n\nAfter providing your explanation, you must output only one of the following choices as your final verdict with a label:\n\n1. Assistant A is significantly better: [[A>>B]]\n2. Assistant A is slightly better: [[A>B]]\n3. Tie, relatively the same: [[A=B]]\n4. Assistant B is slightly better: [[B>A]]\n5. Assistant B is significantly better: [[B>>A]]\n\nExample output: \"My final verdict is tie: [[A=B]]\"."  # noqa: E501
@@ -19,59 +22,77 @@ GRADER_TEMPLATE = "<|User Prompt|>\n{question}\n\n<|The Start of Assistant A's A
 )  # noqa: E501
 
 
-@Benchmark.register(
-    name='general_arena',
-    pretty_name='GeneralArena',
-    tags=['Custom', 'Arena'],
-    description=
-    'GeneralArena is a custom benchmark designed to evaluate the performance of large language models in a competitive setting, '
-    'where models are pitted against each other in custom tasks to determine their relative strengths and weaknesses. You should '
-    'provide the model outputs in the format of a list of dictionaries, where each dictionary contains the model name and its report path. '
-    'For detailed instructions on how to use this benchmark, please refer to the [Arena User Guide](https://evalscope.readthedocs.io/zh-cn/latest/user_guides/arena.html).',
-    dataset_id='general_arena',
-    metric_list=['winrate'],
-    few_shot_num=0,
-    train_split=None,
-    eval_split='test',
-    system_prompt=GRADER_SYSTEM_PROMPT,
-    prompt_template=GRADER_TEMPLATE,
-    extra_params={
-        'models': [{
-            'name': 'qwen-plus',
-            'report_path': 'outputs/20250627_172550/reports/qwen-plus'
-        }, {
-            'name': 'qwen2.5-7b',
-            'report_path': 'outputs/20250627_172817/reports/qwen2.5-7b-instruct'
-        }],
-        'baseline':
-        'qwen2.5-7b'
-    })
-class GeneralArenaAdapter(DataAdapter):
+@register_benchmark(
+    BenchmarkMeta(
+        name='general_arena',
+        pretty_name='GeneralArena',
+        tags=[Tags.CUSTOM, Tags.ARENA],
+        description=
+        'GeneralArena is a custom benchmark designed to evaluate the performance of large language models in a competitive setting, '
+        'where models are pitted against each other in custom tasks to determine their relative strengths and weaknesses. You should '
+        'provide the model outputs in the format of a list of dictionaries, where each dictionary contains the model name and its report path. '
+        'For detailed instructions on how to use this benchmark, please refer to the [Arena User Guide](https://evalscope.readthedocs.io/zh-cn/latest/user_guides/arena.html).',
+        dataset_id='general_arena',
+        metric_list=['winrate'],
+        few_shot_num=0,
+        train_split=None,
+        eval_split='test',
+        system_prompt=GRADER_SYSTEM_PROMPT,
+        prompt_template=GRADER_TEMPLATE,
+        extra_params={
+            'models': [{
+                'name': 'qwen-plus',
+                'report_path': 'outputs/20250627_172550/reports/qwen-plus'
+            }, {
+                'name': 'qwen2.5-7b',
+                'report_path': 'outputs/20250627_172817/reports/qwen2.5-7b-instruct'
+            }],
+            'baseline':
+            'qwen2.5-7b'
+        }
+    )
+)
+class GeneralArenaAdapter(DefaultDataAdapter):
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
 
-        # register metrics
-        metric_registry.register(Metric(name='winrate', object=mean))
+        self._use_llm_judge = True
 
-        # whether to use LLM as a judge
-        self.llm_as_a_judge = True
+        self.models = self.extra_params.get('models', [])
+        self.baseline = self.extra_params.get('baseline', None)
 
-        extra_params = kwargs.get('extra_params', {})
-        self.models = extra_params.get('models', [])
-        self.baseline = extra_params.get('baseline', None)
-
-    def load(self, **kwargs):
+    def load(self):
+        """Load dataset by processing model reports."""
         self._check_names()
         self._check_reports()
         self._check_datasets()
         logger.info(f'Overall datasets: {self.overall_datasets}')
         dataset_model_dict = self._load_common_datasets()
-        data_dict = self._build_pair_wise_data(dataset_model_dict)
-        return data_dict
-
-    def gen_prompt(self, input_d, subset_name, few_shot_list, **kwargs):
-        return self.gen_prompt_data(input_d['question'])
+        datasets = self._build_pair_wise_data(dataset_model_dict)
+
+        # Convert to DatasetDict format
+        dataset_dict = {}
+        for subset_name, samples in datasets.items():
+            dataset = DictDataLoader(
+                dict_list=samples, limit=self.limit, repeats=self.repeats, sample_fields=self.record_to_sample
+            ).load()
+            dataset_dict[subset_name] = dataset
+
+        test_dataset = DatasetDict(dataset_dict)
+        return test_dataset, None
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        """Convert a data record to a Sample object."""
+        return Sample(
+            input=[ChatMessageUser(content=record['question'])],
+            target=record['answer_2'],  # baseline answer
+            metadata={
+                'answer_1': record['answer_1'],
+                'model_1': record['model_1'],
+                'model_2': record['model_2'],
+            }
+        )
 
     def _check_names(self):
         """Check the names of the models and baseline."""
@@ -119,7 +140,8 @@ class GeneralArenaAdapter(DataAdapter):
 
     def _load_common_datasets(self):
         """Load common datasets from the local path."""
-        from evalscope.utils import OutputsStructure, jsonl_to_list
+        from evalscope.utils import OutputsStructure
+        from evalscope.utils.io_utils import jsonl_to_list
 
         dataset_dict = defaultdict(dict)
         for dataset_name, subset_name in self.overall_datasets:
@@ -128,7 +150,8 @@ class GeneralArenaAdapter(DataAdapter):
                 dataset_file_path = os.path.join(dataset_path, f'{dataset_name}_{subset_name}.jsonl')
                 if not os.path.exists(dataset_file_path):
                     raise ValueError(
-                        f'Dataset {dataset_name} with subset {subset_name} not found in model {model["name"]}.')
+                        f'Dataset {dataset_name} with subset {subset_name} not found in model {model["name"]}.'
+                    )
                 dataset = jsonl_to_list(dataset_file_path)
                 # sort by index
                 dataset.sort(key=lambda x: x.get('index'))
@@ -138,9 +161,10 @@ class GeneralArenaAdapter(DataAdapter):
 
     def _build_pair_wise_data(self, dataset_dict):
         """Build pairwise data for the models."""
+        from evalscope.api.evaluator import ReviewResult
         from .utils import process_review_item
 
-        pairwise_data = defaultdict(dict)
+        pairwise_data = defaultdict(list)
         for (dataset_name, subset_name), model_data in dataset_dict.items():
             if len(model_data) < 2:
                 logger.warning(f'Not enough models for dataset {dataset_name} with subset {subset_name}. Skipping.')
@@ -152,8 +176,13 @@ class GeneralArenaAdapter(DataAdapter):
                     continue
                 pairs = []
                 for model_item, baseline_item in zip(model_data[name], model_data[self.baseline]):
+                    # Convert to ReviewResult objects like in get_model_prediction
+                    model_review = ReviewResult.model_validate(model_item)
+                    baseline_review = ReviewResult.model_validate(baseline_item)
+
                     for model_choice, baseline_choice in zip(
-                            process_review_item(model_item), process_review_item(baseline_item)):
+                        process_review_item(model_review), process_review_item(baseline_review)
+                    ):
                         pairs.append({
                             'question': model_choice['Question'],
                             'answer_1': model_choice['Generated'],
@@ -161,23 +190,26 @@ class GeneralArenaAdapter(DataAdapter):
                             'model_1': name,
                             'model_2': self.baseline
                         })
-                pairwise_data[f'{dataset_name}&{subset_name}@{name}&{self.baseline}'][self.eval_split] = pairs
+                pairwise_data[f'{dataset_name}&{subset_name}@{name}&{self.baseline}'] = pairs
 
         return pairwise_data
 
-    def llm_match(self, gold, pred, judge=None, **kwargs):
+    def llm_match_score(
+        self, original_prediction: str, filtered_prediction: str, reference: str, task_state: TaskState
+    ) -> Score:
+        """Use LLM as a judge to evaluate the predicted answer against the baseline."""
         from .utils import get_judge_score, post_process_result
 
-        try:
-            raw_input = kwargs.get('raw_input', None)
-            question = raw_input['question']
-            answer_1 = raw_input['answer_1']
-            answer_2 = raw_input['answer_2']
-            model_1 = raw_input['model_1']
-            model_2 = raw_input['model_2']
-        except KeyError as e:
-            logger.error(f'Missing key in raw input: {e}. Raw input: {raw_input}')
-            raise
+        score = Score(
+            extracted_prediction=filtered_prediction,
+            prediction=original_prediction,
+        )
+
+        question = task_state.input_text
+        answer_1 = task_state.metadata['answer_1']
+        answer_2 = reference  # baseline answer
+        model_1 = task_state.metadata['model_1']
+        model_2 = task_state.metadata['model_2']
 
         system_template = self.system_prompt
         prompt_template = self.prompt_template
@@ -185,9 +217,11 @@ class GeneralArenaAdapter(DataAdapter):
         prompt1 = prompt_template.format(question=question, answer_1=answer_1, answer_2=answer_2)
         # reverse the order
         prompt2 = prompt_template.format(question=question, answer_1=answer_2, answer_2=answer_1)
+
         # get grading response
-        game1_response = judge(prompt1, system_prompt=system_template)
-        game2_response = judge(prompt2, system_prompt=system_template)
+        game1_response = self.llm_judge.judge(prompt1, system_prompt=system_template)
+        game2_response = self.llm_judge.judge(prompt2, system_prompt=system_template)
+
         # parse grading response
         # game1
         res1 = post_process_result(game1_response)
@@ -195,9 +229,9 @@ class GeneralArenaAdapter(DataAdapter):
         # game2
         res2 = post_process_result(game2_response)
         score2 = get_judge_score(res2, reverse=True)
-        return {
-            'score':
-            mean([score1, score2]),
+
+        battle_result = {
+            'score': (score1 + score2) / 2,
             'games': [
                 {
                     'model_a': model_1,
@@ -214,31 +248,38 @@ class GeneralArenaAdapter(DataAdapter):
             ]
         }
 
-    def compute_metric(self, review_res_list: List[dict], **kwargs) -> List[dict]:
-        """
-        compute score of the model
-        """
+        score.value = {'score': battle_result['score']}
+        score.explanation = f'LLM judge battles: Game1: {game1_response[:100]}... Game2: {game2_response[:100]}...'
+        score.metadata = {
+            'source': 'llm_judge',
+            'judge_strategy': getattr(self, 'judge_strategy', 'default'),
+            'model': self.llm_judge.model_id if hasattr(self.llm_judge, 'model_id') else 'unknown',
+            'battle_result': battle_result
+        }
+        score.main_score_name = 'score'
+
+        return score
+
+    def aggregate_scores(self, sample_scores: List[SampleScore]) -> List[AggScore]:
+        """Aggregate scores to compute winrate."""
         import numpy as np
         import pandas as pd
 
         from .utils import compute_mle_elo, get_battles_from_row, get_bootstrap_result, get_win_rate_column
 
-        if isinstance(review_res_list[0], list):
-            review_res_list = [item for sublist in review_res_list for item in sublist]
-
-        battles = pd.concat([get_battles_from_row(res) for res in review_res_list])
+        battles = pd.concat([get_battles_from_row(res.score.metadata['battle_result']) for res in sample_scores])
 
         bt_model_coef = compute_mle_elo(battles, baseline_model=self.baseline)
 
         bootstrap_model_coef = get_bootstrap_result(
-            battles, func_compute_elo=compute_mle_elo, num_round=100, baseline_model=self.baseline)
+            battles, func_compute_elo=compute_mle_elo, num_round=100, baseline_model=self.baseline
+        )
 
         stats = pd.DataFrame()
         stats['results'] = None
         stats['results'] = stats['results'].astype('object')
 
         for i, model in enumerate(bt_model_coef.index):
-            # assert model in bootstrap_elo_lu.columns
             stats.at[i, 'model'] = model
             stats.at[i, 'score'] = bt_model_coef[model]
             stats.at[i, 'lower'] = np.percentile(bootstrap_model_coef[model], 2.5)
@@ -249,20 +290,25 @@ class GeneralArenaAdapter(DataAdapter):
         metrics_dict['winrate_lower'] = get_win_rate_column(stats, 'lower', self.baseline).to_dict()
         metrics_dict['winrate_upper'] = get_win_rate_column(stats, 'upper', self.baseline).to_dict()
 
-        metrics = []
+        agg_scores = []
         for metric_name, models in metrics_dict.items():
-            for model_name, score in models.items():
+            for model_name, score_val in models.items():
                 if model_name == self.baseline:
                     continue
-                metrics.append({'metric_name': metric_name, 'score': score, 'num': len(review_res_list)})
-        return metrics
+                agg_scores.append(AggScore(score=score_val, metric_name=metric_name, num=len(sample_scores)))
+
+        return agg_scores
 
-    def post_process_report(self, report: 'Report', **kwargs):
+    def extract_answer(self, prediction, task_state):
+        # NOTE: This is a hacky way to extract the answer from the prediction
+        return task_state.metadata['answer_1']
+
+    def _on_generate_report_end(self, report: 'Report', output_dir: str, **kwargs):
         """Post-process the report to convert it to a DataFrame with winrate leaderboards."""
         import pandas as pd
         import tabulate
 
-        report_path = kwargs.get('report_path')
+        report_path = output_dir
         leaderboard_file = os.path.join(report_path, 'leaderboard.txt')
 
         # Ensure report directory exists
@@ -288,7 +334,8 @@ class GeneralArenaAdapter(DataAdapter):
             """Format DataFrame as leaderboard with CI."""
             # Pivot to get winrate, winrate_lower, winrate_upper as columns
             pivot_df = data_df.pivot_table(
-                index=[ReportKey.model_name], columns=ReportKey.metric_name, values=ReportKey.score, aggfunc='first')
+                index=[ReportKey.model_name], columns=ReportKey.metric_name, values=ReportKey.score, aggfunc='first'
+            )
 
             # Add baseline model with 50% winrate
             baseline_data = {'winrate': 0.5, 'winrate_lower': 0.5, 'winrate_upper': 0.5}
@@ -392,20 +439,11 @@ class GeneralArenaAdapter(DataAdapter):
             subset_df = parsed_df[(parsed_df['dataset_name'] == dataset_name)
                                   & (parsed_df['subset_name'] == subset_name)]
             leaderboard_outputs.append(
-                format_leaderboard(subset_df, f'=== SUBSET LEADERBOARD: {dataset_name} - {subset_name} ==='))
+                format_leaderboard(subset_df, f'=== SUBSET LEADERBOARD: {dataset_name} - {subset_name} ===')
+            )
 
         # Write all leaderboard outputs to file
         with open(leaderboard_file, 'w', encoding='utf-8') as f:
             f.write('\n'.join(leaderboard_outputs))
 
         logger.info(f'Leaderboard results saved to: {leaderboard_file}')
-
-    def get_gold_answer(self, input_d):
-        return f"model_1: {input_d['model_1']}\n---\n" + input_d['answer_1']
-
-    def llm_parse_pred_result(self, result, raw_input_d=None, eval_type=EvalType.CHECKPOINT):
-        return f"model_2: {raw_input_d['model_2']}\n---\n" + raw_input_d['answer_2']
-
-    def match(self, gold, pred):
-        logger.warning(f'Please use LLMJudge to match the result for {self.name}')
-        return

evalscope/benchmarks/general_arena/utils.py

@@ -7,44 +7,39 @@ from collections import defaultdict
 from sklearn.linear_model import LogisticRegression
 from tqdm import tqdm
 
+from evalscope.api.evaluator import ReviewResult
 from evalscope.utils.logger import get_logger
 
 logger = get_logger()
 
 
-def process_review_item(review_item: dict) -> dict:
+def process_review_item(review_result: ReviewResult) -> list:
     """
-    Process a single review item to extract relevant information.
+    Process a ReviewResult object to extract relevant information.
 
     Args:
-        review_item (dict): The review item to process.
+        review_result: ReviewResult object or dict (for backward compatibility)
 
     Returns:
-        dict: Processed review item with necessary information.
+        list: List of processed review items with necessary information.
     """
-    res = []
-    raw_input = review_item['raw_input']
-    sample_index = review_item['index']
-    question_keys = ['question', 'Question', 'prompt', 'Prompt', 'query', 'Query', 'problem', 'Problem']
-    # Find the first non-empty question key in raw_input
-    question = next((raw_input.get(key) for key in question_keys if raw_input.get(key)), None)
-    for choice_index, choice in enumerate(review_item['choices']):
-        raw_pred_answer = choice['message']['content']
-        parsed_gold_answer = choice['review']['gold']
-        parsed_pred_answer = choice['review']['pred']
-        score = choice['review']['result']
-        raw_d = {
-            'Index': f'{sample_index}_{choice_index}',
-            'Input': raw_input,
-            'Question': question if question else '*No Question*',
-            'Generated': raw_pred_answer,
-            'Gold': parsed_gold_answer if parsed_gold_answer != raw_input else '*Same as Input*',
-            'Pred': parsed_pred_answer,
-            'Score': score,
-        }
-        res.append(raw_d)
 
-    return res
+    # New format using ReviewResult
+    sample_score = review_result.sample_score
+    prediction = sample_score.score.prediction
+    target = review_result.target
+    extracted_prediction = sample_score.score.extracted_prediction
+
+    raw_d = {
+        'Index': str(review_result.index),
+        'Input': review_result.input,
+        'Question': review_result.input,  # Use input as question
+        'Generated': prediction if prediction != extracted_prediction else extracted_prediction,
+        'Gold': target,
+        'Pred': extracted_prediction,
+        'Score': sample_score.score.model_dump(exclude_none=True),
+    }
+    return [raw_d]
 
 
 def post_process_result(completion):
@@ -179,7 +174,8 @@ def compute_mle_elo(df, scale=400, base=10, init_rating=1000, baseline_model='gp
         return elo_scores.sort_values(ascending=False)
 
     lr = LogisticRegression(
-        fit_intercept=False, penalty=None, tol=1e-8)  # May need to set a small value when not use GPT4 as judge model
+        fit_intercept=False, penalty=None, tol=1e-8
+    )  # May need to set a small value when not use GPT4 as judge model
     lr.fit(X, Y)
 
     elo_scores = scale * lr.coef_[0] + init_rating

evalscope/benchmarks/general_mcq/general_mcq_adapter.py

@@ -2,118 +2,57 @@
 import os
 from collections import defaultdict
 
-from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.constants import EvalType, OutputType
-from evalscope.metrics import exact_match
-from evalscope.metrics.completion_parsers import ResponseParser
+from evalscope.api.benchmark import BenchmarkMeta, MultiChoiceAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
 from evalscope.utils.io_utils import csv_to_list, jsonl_to_list
 from evalscope.utils.logger import get_logger
+from evalscope.utils.multi_choices import MultipleChoiceTemplate
 
 # flake8: noqa
 
 logger = get_logger()
 
 
-@Benchmark.register(
-    name='general_mcq',
-    pretty_name='General-MCQ',
-    description='A general multiple-choice question answering dataset for custom evaluation. '
-    'For detailed instructions on how to use this benchmark, please refer to the [User Guide](https://evalscope.readthedocs.io/zh-cn/latest/advanced_guides/custom_dataset/llm.html#mcq).',
-    tags=['MCQ', 'Custom'],
-    dataset_id='general_mcq',
-    model_adapter=OutputType.GENERATION,
-    output_types=[OutputType.MULTIPLE_CHOICE, OutputType.GENERATION],
-    subset_list=['default'],
-    metric_list=['AverageAccuracy'],
-    few_shot_num=0,
-    train_split='dev',
-    eval_split='val',
-    prompt_template='请回答问题，并选出其中的正确答案。你的回答的最后一行应该是这样的格式：“答案是：LETTER”（不带引号），其中 LETTER 是 A、B、C、D 中的一个。\n{query}',
-    query_template='问题：{question}\n{choices}\n答案: {answer}\n\n')
-class GeneralMCQAdapter(DataAdapter):
+@register_benchmark(
+    BenchmarkMeta(
+        name='general_mcq',
+        pretty_name='General-MCQ',
+        description='A general multiple-choice question answering dataset for custom evaluation. '
+        'For detailed instructions on how to use this benchmark, please refer to the [User Guide](https://evalscope.readthedocs.io/zh-cn/latest/advanced_guides/custom_dataset/llm.html#mcq).',
+        tags=[Tags.MULTIPLE_CHOICE, Tags.CUSTOM],
+        dataset_id='general_mcq',
+        subset_list=['default'],
+        metric_list=['acc'],
+        few_shot_num=0,
+        train_split='dev',
+        eval_split='val',
+        prompt_template=MultipleChoiceTemplate.CHINESE_SINGLE_ANSWER_TEMPLATE,
+    )
+)
+class GeneralMCQAdapter(MultiChoiceAdapter):
 
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
 
         self.choices = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J']
 
-    def load_from_disk(self, dataset_name_or_path, subset_list, work_dir, **kwargs) -> dict:
-        data_dict = defaultdict(dict)
-        for subset_name in subset_list:
-            for split_name in [self.train_split, self.eval_split]:
-                # Check for files with different extensions
-                for ext, loader in [('.jsonl', jsonl_to_list), ('.csv', csv_to_list)]:
-                    if os.path.exists(dataset_name_or_path):
-                        file_path = os.path.join(dataset_name_or_path, f'{subset_name}_{split_name}{ext}')
-                    else:
-                        file_path = os.path.join(work_dir, dataset_name_or_path, f'{subset_name}_{split_name}{ext}')
-
-                    if os.path.exists(file_path):
-                        data_dict[subset_name][split_name] = loader(file_path)
-                        break  # Stop checking other extensions once a file is found
-
-        return dict(data_dict)
-
-    def gen_prompt(self, input_d: dict, subset_name: str, few_shot_list: list, **kwargs) -> dict:
-        """
-        Generate model prompt from raw input, unify the prompt format for C-Eval benchmark.
-
-        Args:
-            input_d (dict): The raw input. A single data format of the C-Eval:
-
-            {'id': 0,
-            'question': '下列关于税法基本原则的表述中，不正确的是____。',
-            'A': '税收法定原则包括税收要件法定原则和税务合法性原则',
-            'B': '税收公平原则源于法律上的平等性原则',
-            'C': '税收效率原则包含经济效率和行政效率两个方面',
-            'D': '税务机关按法定程序依法征税，可以自由做出减征、停征或免征税款的决定',
-            'answer': 'D'}
-
-        Returns:
-            {'data': ['prompt ...']}
-        """
-
-        few_shot_prompts = [self._format_example(input_d=sample, include_answer=True) for sample in few_shot_list]
-
-        if len(few_shot_prompts) > 0:
-            context: str = '\n'.join(few_shot_prompts) + '\n'
-        else:
-            context = ''
-        context = context.strip() + self._format_example(input_d=input_d, include_answer=False)
-
-        full_prompt = self.prompt_template.format(query=context)
-
-        return self.gen_prompt_data(full_prompt)
-
-    def get_gold_answer(self, input_d: dict) -> str:
-        # Get the gold choice
-        return input_d.get('answer', '')
-
-    def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = EvalType.CHECKPOINT) -> str:
-        """
-        Parse the model output to get the answer. Could be the best choice index.
-
-        Args:
-            result: Predicted answer from the model. Usually a string for chat.
-            raw_input_d (dict): The raw input. Depending on the dataset.
-            eval_type: `checkpoint` or `service` or `custom`. Default is `checkpoint`.
-
-        Returns:
-            The parsed answer. Depending on the dataset. Usually a string for chat.
-        """
-        if self.model_adapter == OutputType.MULTIPLE_CHOICE:
-            return result
-        else:
-            return ResponseParser.parse_first_option_with_choices(text=result, options=self.choices)
-
-    def match(self, gold: str, pred: str) -> float:
-        return exact_match(gold=gold, pred=pred)
-
-    def _format_example(self, input_d: dict, include_answer=True):
-        choices_str = '\n'.join([f'{choice}. {input_d[choice]}' for choice in self.choices if choice in input_d])
-
-        if include_answer:
-            return self.query_template.format(
-                question=input_d['question'], choices=choices_str, answer=input_d['answer'])
-        else:
-            return self.query_template.format(question=input_d['question'], choices=choices_str, answer='').rstrip()
+    def load_from_disk(self, **kwargs):
+        return super().load_from_disk(use_local_loader=True)
+
+    def record_to_sample(self, record) -> Sample:
+        # Extract choices from the record (A, B, C, D, etc.)
+        choices = []
+        for choice_key in self.choices:
+            if choice_key in record:
+                choices.append(record[choice_key])
+            else:
+                break  # Stop when we reach a choice key that doesn't exist
+
+        return Sample(
+            input=record['question'],
+            choices=choices,
+            target=record['answer'],
+            metadata={'id': record.get('id', 'unknown')},
+        )