evalscope 0.10.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

evalscope/benchmarks/blink/blink_adapter.py

@@ -0,0 +1,61 @@
+import re
+from typing import Any, Dict, List
+
+from evalscope.api.benchmark import BenchmarkMeta, MultiChoiceAdapter, VisionLanguageAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.messages import ChatMessageUser, Content, ContentImage, ContentText
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.io_utils import bytes_to_base64
+from evalscope.utils.logger import get_logger
+from evalscope.utils.multi_choices import format_letter_choices
+
+logger = get_logger()
+
+MULT_CHOICE_PROMPT = r"""
+Answer the following multiple choice question. The last line of your response should be of the following format:
+'ANSWER: $LETTER' (without quotes) where LETTER is one of {letters}.
+
+{question}
+""".strip()
+
+SUBSET_LIST = [
+    'Art_Style', 'Counting', 'Forensic_Detection', 'Functional_Correspondence', 'IQ_Test', 'Jigsaw',
+    'Multi-view_Reasoning', 'Object_Localization', 'Relative_Depth', 'Relative_Reflectance', 'Semantic_Correspondence',
+    'Spatial_Relation', 'Visual_Correspondence', 'Visual_Similarity'
+]
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='blink',
+        pretty_name='BLINK',
+        tags=[Tags.MULTI_MODAL, Tags.KNOWLEDGE, Tags.MULTIPLE_CHOICE],
+        description=
+        'BLINK is a benchmark designed to evaluate the core visual perception abilities of multimodal large language models (MLLMs). It transforms 14 classic computer vision tasks into 3,807 multiple-choice questions, accompanied by single or multiple images and visual prompts.',  # noqa: E501
+        dataset_id='evalscope/BLINK',
+        subset_list=SUBSET_LIST,
+        metric_list=['acc'],
+        eval_split='val',
+        prompt_template=MULT_CHOICE_PROMPT,
+    )
+)
+class BLINKAdapter(VisionLanguageAdapter, MultiChoiceAdapter):
+    MAX_IMAGES: int = 4
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        choices = record.get('choices')
+        input_text = MULT_CHOICE_PROMPT.format(question=record['prompt'], letters=format_letter_choices(choices))
+        content_list: List[Content] = [ContentText(text=input_text)]
+
+        for i in range(1, self.MAX_IMAGES + 1):
+            image = record.get(f'image_{i}')
+            if image:
+                image_base64 = bytes_to_base64(image['bytes'], format='jpeg', add_header=True)
+                content_list.append(ContentImage(image=image_base64))
+
+        label_answer = record['answer'].strip('(').strip(')')
+        return Sample(input=[ChatMessageUser(content=content_list)], choices=choices, target=label_answer)

evalscope/benchmarks/ceval/ceval_adapter.py

@@ -1,73 +1,14 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
-import csv
-import os
-
-from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.constants import EvalType
-from evalscope.metrics import AverageAccuracy
-from evalscope.metrics.metrics import exact_match, weighted_mean
-from evalscope.models import MultiChoiceModelAdapter
-from evalscope.utils import ResponseParser, normalize_score
-from evalscope.utils.logger import get_logger
 
-# flake8: noqa
+from typing import Any, Dict
 
-logger = get_logger()
+from evalscope.api.benchmark import BenchmarkMeta, MultiChoiceAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.logger import get_logger
 
-SUBSET_LIST = [
-    'computer_network',
-    'operating_system',
-    'computer_architecture',
-    'college_programming',
-    'college_physics',
-    'college_chemistry',
-    'advanced_mathematics',
-    'probability_and_statistics',
-    'discrete_mathematics',
-    'electrical_engineer',
-    'metrology_engineer',
-    'high_school_mathematics',
-    'high_school_physics',
-    'high_school_chemistry',
-    'high_school_biology',
-    'middle_school_mathematics',
-    'middle_school_biology',
-    'middle_school_physics',
-    'middle_school_chemistry',
-    'veterinary_medicine',
-    'college_economics',
-    'business_administration',
-    'marxism',
-    'mao_zedong_thought',
-    'education_science',
-    'teacher_qualification',
-    'high_school_politics',
-    'high_school_geography',
-    'middle_school_politics',
-    'middle_school_geography',
-    'modern_chinese_history',
-    'ideological_and_moral_cultivation',
-    'logic',
-    'law',
-    'chinese_language_and_literature',
-    'art_studies',
-    'professional_tour_guide',
-    'legal_professional',
-    'high_school_chinese',
-    'high_school_history',
-    'middle_school_history',
-    'civil_servant',
-    'sports_science',
-    'plant_protection',
-    'basic_medicine',
-    'clinical_medicine',
-    'urban_and_rural_planner',
-    'accountant',
-    'fire_engineer',
-    'environmental_impact_assessment_engineer',
-    'tax_accountant',
-    'physician',
-]
+logger = get_logger()
 
 SUBJECT_MAPPING = {
     'computer_network': ['Computer Network', '计算机网络', 'STEM'],
@@ -124,127 +65,105 @@ SUBJECT_MAPPING = {
     'physician': ['Physician', '医师资格', 'Other']
 }
 
-
-@Benchmark.register(
-    name='ceval',
-    dataset_id='modelscope/ceval-exam',
-    model_adapter=MultiChoiceModelAdapter,
-    subset_list=SUBSET_LIST,
-    metric_list=[AverageAccuracy],
-    few_shot_num=0,
-    train_split='dev',
-    eval_split='val',
+# Based on the prompt template for Chinese evaluation
+USER_PROMPT_TEMPLATE = """以下是中国关于{subject}的单项选择题，请选出其中的正确答案。你的回答的最后一行应该是这样的格式："答案：LETTER"（不带引号），其中 LETTER 是 A、B、C、D 中的一个。
+
+问题：{question}
+选项：
+{choices}
+""".lstrip()  # noqa: E501
+
+FEWSHOT_TEMPLATE = """以下是一些示例问题：
+
+{fewshot}
+
+""".lstrip()
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='ceval',
+        pretty_name='C-Eval',
+        tags=[Tags.KNOWLEDGE, Tags.MULTIPLE_CHOICE, Tags.CHINESE],
+        description=
+        'C-Eval is a benchmark designed to evaluate the performance of AI models on Chinese exams across various subjects, including STEM, social sciences, and humanities. It consists of multiple-choice questions that test knowledge and reasoning abilities in these areas.',  # noqa: E501
+        dataset_id='evalscope/ceval',
+        subset_list=list(SUBJECT_MAPPING.keys()),
+        metric_list=['acc'],
+        few_shot_num=5,
+        train_split='dev',
+        eval_split='val',
+        prompt_template=USER_PROMPT_TEMPLATE,
+        few_shot_prompt_template=FEWSHOT_TEMPLATE,
+    )
 )
-class CEVALAdapter(DataAdapter):
-
-    choices = ['A', 'B', 'C', 'D']
+class CEVALAdapter(MultiChoiceAdapter):
 
     def __init__(self, **kwargs):
 
-        few_shot_num = kwargs.get('few_shot_num', 0)
-        if few_shot_num > 5:
-            logger.warning(f'few_shot_num <= 5 for C-Eval, but got {few_shot_num}. Use 5-shot by default.')
-            kwargs['few_shot_num'] = 5
         super().__init__(**kwargs)
 
         self.category_map = {k: v[-1] for k, v in SUBJECT_MAPPING.items()}
 
-    def load_from_disk(self, dataset_name_or_path, subset_list, work_dir, **kwargs) -> dict:
-        data_dict = {}
-        for subset_name in subset_list:
-            for split_name in [self.train_split, self.eval_split]:
-                if os.path.exists(dataset_name_or_path):
-                    file_path = os.path.join(dataset_name_or_path, f'{subset_name}_{split_name}.csv')
-                else:
-                    file_path = os.path.join(work_dir, dataset_name_or_path, f'{subset_name}_{split_name}.csv')
-                if os.path.exists(file_path):
-                    with open(file_path, encoding='utf-8') as f:
-                        rows = []
-                        reader = csv.reader(f)
-                        header = next(reader)
-                        for row in reader:
-                            item = dict(zip(header, row))
-                            item.setdefault('explanation', '')
-                            item.setdefault('answer', '')
-                            rows.append(item)
-
-                        if subset_name in data_dict:
-                            data_dict[subset_name].update({split_name: rows})
-                        else:
-                            data_dict[subset_name] = {split_name: rows}
-
-        return data_dict
-
-    def gen_prompt(self, input_d: dict, subset_name: str, few_shot_list: list, **kwargs) -> dict:
-        """
-        Generate model prompt from raw input, unify the prompt format for C-Eval benchmark.
-
-        Args:
-            input_d (dict): The raw input. A single data format of the C-Eval:
-
-            {'id': 0,
-            'question': '下列关于税法基本原则的表述中，不正确的是____。',
-            'A': '税收法定原则包括税收要件法定原则和税务合法性原则',
-            'B': '税收公平原则源于法律上的平等性原则',
-            'C': '税收效率原则包含经济效率和行政效率两个方面',
-            'D': '税务机关按法定程序依法征税，可以自由做出减征、停征或免征税款的决定',
-            'answer': 'D',
-            'explanation': ''}
-
-        Returns:
-            {'data': ['prompt ...']}
-        """
-
-        few_shot_prompts = [self._format_example(input_d=sample, include_answer=True) for sample in few_shot_list]
-
-        if len(few_shot_prompts) > 0:
-            context: str = '\n'.join(few_shot_prompts) + '\n'
-        else:
-            context = ''
-
-        full_prompt: str = context.strip() + self._format_example(input_d=input_d, include_answer=False)
-
-        subject_name: str = SUBJECT_MAPPING.get(subset_name)[1] if SUBJECT_MAPPING.get(subset_name) else subset_name
-        full_prompt = f'以下是中国关于{subject_name}考试的单项选择题，请选出其中的正确答案。\n' + full_prompt
-
-        return {'data': [full_prompt], 'multi_choices': self.choices, 'system_prompt': self.prompt_template}
-
-    def get_gold_answer(self, input_d: dict) -> str:
-        # Get the gold choice
-        return input_d.get('answer', '')
-
-    def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = EvalType.CHECKPOINT) -> str:
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        # Build choices list from A, B, C, D fields
+        choices = [record['A'], record['B'], record['C'], record['D']]
+        subset = self.current_subset_name
+
+        return Sample(
+            input=record['question'],
+            choices=choices,
+            target=record['answer'],
+            metadata={
+                'id': record.get('id', ''),
+                'explanation': record.get('explanation', ''),
+                'subject': subset
+            },
+        )
+
+    def sample_to_fewshot(self, sample: Sample) -> str:
+        q_str = f"""问题：{sample.input}"""
+        choices = sample.choices if sample.choices is not None else []
+        opt_str_list = []
+        for i, choice in enumerate(choices):
+            opt_str_list.append(f"""{chr(65 + i)}. {choice}""")
+        opt_str = '\n'.join(opt_str_list)
+        opt_str = f"""选项：\n{opt_str}"""
+        exp_str = f"""解析：{sample.metadata.get('explanation', '')}"""
+        ans_str = f"""答案：{sample.target}"""
+        final_str = '\n'.join([q_str, opt_str, exp_str, ans_str])
+
+        return final_str
+
+    def format_fewshot_template(self, fewshot, sample):
+        fewshot_str = FEWSHOT_TEMPLATE.format(fewshot=fewshot)
+        prompt_str = self.format_prompt_template(sample)
+        return fewshot_str + '\n' + prompt_str
+
+    def format_prompt_template(self, sample):
+        subject_name = SUBJECT_MAPPING.get(sample.metadata['subject'])[1]
+        choices = sample.choices if sample.choices is not None else []
+        choices_str = '\n'.join([f'{chr(65 + i)}. {choice}' for i, choice in enumerate(choices)])
+
+        return USER_PROMPT_TEMPLATE.format(subject=subject_name, question=sample.input, choices=choices_str)
+
+    def extract_answer(self, prediction, task_state) -> str:
         """
-        Parse the model output to get the answer. Could be the best choice index.
+        Extract the answer from the prediction based on the task state.
 
         Args:
-            result: Predicted answer from the model. Usually a string for chat.
-            raw_input_d (dict): The raw input. Depending on the dataset.
-            eval_type: `checkpoint` or `service` or `custom`. Default is `checkpoint`.
+            prediction (str): The model's prediction string
+            task_state (dict): The current task state containing metadata
 
         Returns:
-            The parsed answer. Depending on the dataset. Usually a string for chat.
+            str: The extracted answer from the prediction
         """
-        if eval_type == EvalType.CHECKPOINT:
-            return result
-        elif eval_type == EvalType.SERVICE:
-            return ResponseParser.parse_first_option_with_choices(result, self.choices)  # TODO: to be checked !
-        elif eval_type == EvalType.CUSTOM:
-            return ResponseParser.parse_first_option_with_choices(result, self.choices)  # TODO: to be checked !
-        else:
-            raise ValueError(f'Invalid eval_type: {eval_type}')
-
-    def match(self, gold: str, pred: str) -> float:
-        return exact_match(gold=gold, pred=pred)
-
-    @classmethod
-    def _format_example(cls, input_d: dict, include_answer=True):
-        example = '问题：' + input_d['question']
-        for choice in cls.choices:
-            example += f'\n{choice}. {input_d[f"{choice}"]}'
+        import re
 
-        if include_answer:
-            example += '\n答案: ' + input_d['answer'] + '\n\n'
+        # Use regex to find the answer in the format "答案：LETTER"
+        match = re.search(r'答案：([A-D])', prediction)
+        if match:
+            return match.group(1)
         else:
-            example += '\n答案: '
-        return example
+            logger.warning(f'No valid answer found in prediction: {prediction}')
+            return ''

evalscope/benchmarks/chartqa/chartqa_adapter.py

@@ -0,0 +1,80 @@
+import re
+from typing import Any, Dict, List
+
+from evalscope.api.benchmark import BenchmarkMeta, VisionLanguageAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.evaluator import TaskState
+from evalscope.api.messages import ChatMessageUser, Content, ContentImage, ContentText
+from evalscope.api.metric.scorer import Score
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.io_utils import bytes_to_base64
+from evalscope.utils.logger import get_logger
+
+# flake8: noqa
+
+logger = get_logger()
+
+OPEN_PROMPT = """
+{question}
+
+The last line of your response should be of the form "ANSWER: $ANSWER" (without quotes) where $ANSWER is the a single word answer to the problem.
+"""
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='chartqa',
+        pretty_name='ChartQA',
+        tags=[Tags.MULTI_MODAL, Tags.KNOWLEDGE, Tags.QA],
+        description=
+        'ChartQA is a benchmark designed to evaluate question-answering capabilities about charts (e.g., bar charts, line graphs, pie charts), focusing on both visual and logical reasoning.',  # noqa: E501
+        dataset_id='lmms-lab/ChartQA',
+        subset_list=['human_test', 'augmented_test'],
+        metric_list=['relaxed_acc'],
+        eval_split='test',
+        prompt_template=OPEN_PROMPT,
+    )
+)
+class ChartQAAdapter(VisionLanguageAdapter):
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        self.add_aggregation_name = False
+        self.reformat_subset = True
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        question = record['question']
+        image_data = record['image']
+        image_base64 = bytes_to_base64(image_data['bytes'], format='png', add_header=True)
+
+        content_list: List[Content] = [
+            ContentText(text=OPEN_PROMPT.format(question=question)),
+            ContentImage(image=image_base64)
+        ]
+
+        return Sample(
+            input=[ChatMessageUser(content=content_list)],
+            target=record['answer'],
+            subset_key=record['type'],  # 'human_test' or 'augmented_split'
+        )
+
+    def extract_answer(self, prediction: str, task_state: TaskState) -> str:
+        pattern = r'ANSWER:\s*(.*)'
+        match = re.search(pattern, prediction)
+        if match:
+            return match.group(1).strip()
+        return ''
+
+    def match_score(self, original_prediction, filtered_prediction, reference, task_state) -> Score:
+        from .utils import relaxed_correctness
+
+        score = relaxed_correctness(filtered_prediction, reference)
+        score = 1.0 if score else 0.0
+
+        return Score(
+            value={'relaxed_acc': score},
+            prediction=original_prediction,
+            extracted_prediction=filtered_prediction,
+        )

evalscope/benchmarks/chartqa/utils.py

@@ -0,0 +1,38 @@
+def relaxed_correctness(prediction: str, target: str, max_relative_change: float = 0.05) -> bool:
+    """Calculates relaxed correctness.
+
+    The correctness tolerates certain error ratio defined by max_relative_change.
+    See https://arxiv.org/pdf/2203.10244.pdf, end of section 5.1:
+    “Following Methani et al. (2020), we use a relaxed accuracy measure for the
+    numeric answers to allow a minor inaccuracy that may result from the automatic
+    data extraction process. We consider an answer to be correct if it is within
+    5% of the gold answer. For non-numeric answers, we still need an exact match
+    to consider an answer to be correct.”
+
+    This funcion is taken from https://github.com/QwenLM/Qwen-VL/blob/34b4c0ee7b07726371b960911f249fe61b362ca3/eval_mm/evaluate_vqa.py#L113
+    Args:
+      target: List of target string.
+      prediction: List of predicted string.
+      max_relative_change: Maximum relative change.
+
+    Returns:
+      Whether the prediction was correct given the specified tolerance.
+    """  # noqa: E501
+
+    def _to_float(text: str):
+        try:
+            if text.endswith('%'):
+                # Convert percentages to floats.
+                return float(text.rstrip('%')) / 100.0
+            else:
+                return float(text)
+        except ValueError:
+            return None
+
+    prediction_float = _to_float(prediction)
+    target_float = _to_float(target)
+    if prediction_float is not None and target_float:
+        relative_change = abs(prediction_float - target_float) / abs(target_float)
+        return relative_change <= max_relative_change
+    else:
+        return prediction.lower() == target.lower()

evalscope/benchmarks/chinese_simple_qa/csimple_qa_adapter.py

@@ -0,0 +1,170 @@
+import re
+from typing import Any, Dict
+
+from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.evaluator import TaskState
+from evalscope.api.metric import Score
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+GRADER_TEMPLATE = """
+请根据给定问题、标准答案和模型预测的答案来评估模型的回答是否正确。您的任务是将结果评定为：【正确】、【错误】或【未尝试】。
+
+首先，我们将列出每个评定类别的示例，然后请您对新问题的预测答案进行评定。
+以下是【正确】的答复示例：
+```
+问题：贝拉克·奥巴马的孩子叫什么名字？
+标准答案：玛丽亚·奥巴马和萨莎·奥巴马
+模型预测1：Malia Obama and Sasha Obama
+模型预测2：玛丽亚和萨沙
+模型预测3：大多数人会说是玛丽亚和萨莎，但我不确定，需要再确认
+模型预测4：巴拉克·奥巴马有两个女儿，她们分别是玛丽亚·安和娜塔莎·玛丽安，但通常称作玛丽亚·奥巴马和萨莎·奥巴马。玛丽亚出生于1998年7月4日，萨莎出生于2001年6月10日。
+```
+这些答复均为【正确】，因为：
+    - 完整地包含了标准答案中的重要信息。
+    - 不包含任何与标准答案矛盾的信息。
+    - 只关注语义内容，中英文，大小写、标点、语法和顺序不重要。
+    - 答复中出现模糊语句或猜测是可以接受的，前提是包含了标准答案且不含有不正确信息或矛盾。
+
+以下是【错误】的答复示例：
+```
+问题：巴拉克·奥巴马的孩子叫什么名字？
+标准答案：玛丽亚·奥巴马和萨莎·奥巴马
+模型预测1：玛丽亚
+模型预测2：玛丽亚、萨莎和苏珊
+模型预测3：巴拉克·奥巴马没有孩子
+模型预测4：我认为是玛丽亚和萨莎。或者是玛丽亚和杰基。或者是乔伊和玛丽亚。
+模型预测5：虽然我不知道他们的确切名字，但能说出巴拉克·奥巴马有三个孩子。
+模型预测6：你可能是想说贝茜和奥利维亚。不过您应通过最新的参考资料确认详细信息。那是正确的答案吗？
+```
+这些答复均为【错误】，因为：
+    - 答复中包含与标准答案矛盾的事实陈述。即使在陈述中略带保留（例如："可能是"，"虽然我不确定，但我认为"），也视为错误。
+
+以下是【未尝试】的答复示例：
+```
+问题：巴拉克·奥巴马的孩子叫什么名字？
+标准答案：玛丽亚·奥巴马和萨莎·奥巴马
+模型预测1：我不知道。
+模型预测2：我需要更多关于您所指奥巴马的上下文。
+模型预测3：不查阅网络我无法回答这个问题，不过我知道巴拉克·奥巴马有两个孩子。
+模型预测4：巴拉克·奥巴马有两个孩子。我知道其中一个叫玛丽亚，但我不确定另一个的名字。
+```
+这些答复均为【未尝试】，因为：
+    - 没有包含标准答案中的重要信息。
+    - 回复中没有与标准答案矛盾的陈述。
+
+另外注意以下几点：
+- 对于标准答案为数字的问题，预测答案应和标准答案一致。例如，考虑问题"金山铁路黄浦江特大桥的全长是多少米？"，标准答案为"3518.17"：
+    - 预测答案"3518"、"3518.1"、"3518.17"均为【正确】。
+    - 预测答案"3520"和"3600"均为【错误】。
+    - 预测答案"大约3500米"和"超过3000米"被视为【未尝试】，因为它们既不确认也不与标准答案矛盾。
+- 如果标准答案包含比问题更多的信息，预测答案只需包含问题中提到的信息。
+    - 例如，考虑问题"菱镁矿的主要化学成分是什么？"标准答案为"碳酸镁（MgCO3）"。"碳酸镁"或"MgCO3"均视为【正确】答案。
+- 如果从问题中明显可以推断出预测答案省略的信息，那么算作正确。
+    - 例如，问题"巴鲁米尼的努拉吉遗迹在1997年被联合国教科文组织列为世界文化遗产，那么这遗址在哪个地区？"标准答案为"意大利撒丁岛"，预测答案"撒丁岛"被视为【正确】。
+- 如果能明显看出名字翻译版本不同但是是同一个人也认为正确。
+    - 例如，如果标准答案是"Robinson"，那么回答鲁滨逊或者鲁滨孙均正确。
+
+下面是一个新的问题示例。请只回复A、B、C之一，不要道歉或纠正自己的错误，只需要评估该回答。
+```
+问题: {question}
+正确答案: {target}
+预测答案: {predicted_answer}
+```
+
+将此新问题的预测答案评定为以下之一：
+A:【正确】
+B:【错误】
+C:【未尝试】
+
+只返回字母"A"、"B"或"C"，无须添加其他文本。
+""".strip()
+
+SUBSET_LIST = ['中华文化', '人文与社会科学', '工程、技术与应用科学', '生活、艺术与文化', '社会', '自然与自然科学']
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='chinese_simpleqa',
+        pretty_name='Chinese-SimpleQA',
+        tags=[Tags.KNOWLEDGE, Tags.QA, Tags.CHINESE],
+        description=
+        "Chinese SimpleQA is a Chinese question-answering dataset designed to evaluate the performance of language models on simple factual questions. It includes a variety of topics and is structured to test the model's ability to understand and generate correct answers in Chinese.",  # noqa: E501
+        subset_list=SUBSET_LIST,
+        dataset_id='AI-ModelScope/Chinese-SimpleQA',
+        metric_list=['is_correct', 'is_incorrect', 'is_not_attempted'],
+        few_shot_num=0,
+        train_split=None,
+        eval_split='train',
+        prompt_template='请回答问题：\n\n{question}'
+    )
+)
+class ChineseSimpleQAAdapter(DefaultDataAdapter):
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        self._use_llm_judge = True  # Use LLM as a judge by default
+        self.reformat_subset = True  # Reformat subset to primary_category
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        """
+        Convert a data record to a Sample object.
+
+        Args:
+            record (Dict[str, Any]): Input data record.
+
+        Returns:
+            Sample: Sample object with input, target, and metadata.
+        """
+        question = record['question']
+        answer = record['answer']
+        subset_key = record.get('primary_category', 'default')  # Use primary_category as subset key
+        metadata = {
+            'id': record.get('id', 'unknown'),
+            'primary_category': subset_key,
+            'secondary_category': record.get('secondary_category', '')
+        }
+
+        return Sample(input=question, target=answer, subset_key=subset_key, metadata=metadata)
+
+    def llm_match_score(
+        self,
+        original_prediction: str,
+        filtered_prediction: str,
+        reference: str,
+        task_state: TaskState,
+    ) -> Score:
+        score = Score(
+            extracted_prediction=filtered_prediction,
+            prediction=original_prediction,
+        )
+
+        question = task_state.input_text
+
+        # Request judge and obtain score
+        prompt = GRADER_TEMPLATE.format(question=question, target=reference, predicted_answer=filtered_prediction)
+        system_prompt = '你是一个智能助手，请根据给定问题、标准答案和模型预测的答案来评估模型的回答是否正确。'
+        judge_response = self.llm_judge.judge(prompt, system_prompt=system_prompt)
+        # parse grading response
+        match = re.search(r'(A|B|C)', judge_response)
+        res = match.group(0) if match else 'C'
+
+        # Set score based on the match result
+        score.value = {
+            'is_correct': 1 if res == 'A' else 0,
+            'is_incorrect': 1 if res == 'B' else 0,
+            'is_not_attempted': 1 if res == 'C' else 0,
+        }
+        score.explanation = f'LLM judge: {judge_response}'
+        score.metadata = {
+            'source': 'llm_judge',
+            'judge_strategy': self.judge_strategy,
+            'model': self.llm_judge.model_id
+        }
+        score.main_score_name = 'is_correct'
+        return score