evalscope 0.17.1__py3-none-any.whl → 1.0.0__py3-none-any.whl

evalscope/benchmarks/gsm8k/gsm8k_adapter.py

@@ -1,156 +1,76 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
-# Copyright (c) EleutherAI, Inc. and its affiliates.
-# flake8: noqa
-import math
-import os
-import re
-
-from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.utils.io_utils import jsonl_to_list
+
+from typing import Any, Dict
+
+from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.evaluator import TaskState
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
 from evalscope.utils.logger import get_logger
 
 logger = get_logger()
 
+PROMPT_TEMPLATE = """
+Solve the following math problem step by step. The last line of your response should be of the form "ANSWER: $ANSWER" (without quotes) where $ANSWER is the answer to the problem.
+
+{question}
+
+Remember to put your answer on its own line at the end in the form "ANSWER: $ANSWER" (without quotes) where $ANSWER is the answer to the problem, and you do not need to use a \\boxed command.
+
+Reasoning:
+""".lstrip()  # noqa: E501
+
+FEWSHOT_TEMPLATE = """
+Here are some examples of how to solve similar problems:
+
+{fewshot}
 
-@Benchmark.register(
-    name='gsm8k',
-    pretty_name='GSM8K',
-    tags=['Mathematics'],
-    description=
-    'GSM8K (Grade School Math 8K) is a dataset of grade school math problems, designed to evaluate the mathematical reasoning abilities of AI models.',
-    dataset_id='modelscope/gsm8k',
-    subset_list=['main'],
-    metric_list=['AverageAccuracy'],
-    few_shot_num=4,
-    train_split=None,
-    eval_split='test',
-    prompt_template="Question: {query}\nLet's think step by step\nAnswer:",
+""".lstrip() + PROMPT_TEMPLATE
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='gsm8k',
+        pretty_name='GSM8K',
+        dataset_id='AI-ModelScope/gsm8k',
+        tags=[Tags.MATH, Tags.REASONING],
+        description=
+        'GSM8K (Grade School Math 8K) is a dataset of grade school math problems, designed to evaluate the mathematical reasoning abilities of AI models.',  # noqa: E501
+        subset_list=['main'],
+        few_shot_num=4,
+        train_split='train',
+        eval_split='test',
+        metric_list=['acc'],
+        prompt_template=PROMPT_TEMPLATE,
+        few_shot_prompt_template=FEWSHOT_TEMPLATE,
+    )
 )
-class GSM8KAdapter(DataAdapter):
+class GSM8KAdapter(DefaultDataAdapter):
 
     def __init__(self, **kwargs):
-        """
-        Data adapter for GSM8K dataset.
-
-        Args:
-            subset_list (list): Subset list for the dataset. Default: ['main']
-            metric_list (list): Metric list for the dataset. Default: [{'name': 'AverageAccuracy', 'object': mean}]
-            few_shot_num (int): Number of few-shot examples. Default: 4
-            train_split (str): Train split name. Default: 'train'
-            eval_split (str): The target eval split name. Default: 'test'
-            **kwargs: ...
-        """
-        few_shot_num = kwargs.get('few_shot_num', 4)
-        if few_shot_num != 4 and few_shot_num != 0:
-            logger.error(f'GSM8K uses 4-shot examples with CoT or 0-shot by system, but got {few_shot_num}. '
-                         f'Use 4-shot by default.')
-            kwargs['few_shot_num'] = 4
-
         super().__init__(**kwargs)
 
-    def load_from_disk(self, dataset_name_or_path, subset_list, work_dir, **kwargs) -> dict:
-        data_dict = {}
-        for subset_name in subset_list:
-            data_dict[subset_name] = {}
-            for split in [self.train_split, self.eval_split]:
-                if os.path.exists(dataset_name_or_path):
-                    file_path = os.path.join(dataset_name_or_path, f'{split}.jsonl')
-                else:
-                    file_path = os.path.join(work_dir, dataset_name_or_path, f'{split}.jsonl')
-                if os.path.exists(file_path):
-                    data_dict[subset_name][split] = jsonl_to_list(file_path)
-
-        return data_dict
-
-    def gen_prompt(self, input_d: dict, few_shot_list: list, **kwargs) -> dict:
-        """
-        Generate prompt for the model.
-
-        Args:
-            input_d (dict): The raw input. A single data format of the GSM8K:
-            {
-                "question": "Janet\\u2019s ducks lay 16 eggs per day. She eats three for breakfast every morning and bakes muffins for her friends every day with four. She sells the remainder at the farmers\' market daily for $2 per fresh duck egg. How much in dollars does she make every day at the farmers\' market?",
-                "answer": "Janet sells 16 - 3 - 4 = <<16-3-4=9>>9 duck eggs a day.\\nShe makes 9 * 2 = $<<9*2=18>>18 every day at the farmer\\u2019s market.\\n#### 18"
-            }
-        """
-        use_fewshot = self.few_shot_num > 0
-        context = self._generate_prompt(use_fewshot=use_fewshot)
-
-        full_prompt = context + self.prompt_template.format(query=input_d['question'])
-
-        return self.gen_prompt_data(full_prompt)
-
-    def get_gold_answer(self, input_d: dict) -> str:
-        # Extract the gold answer from the input dict.
-        ans: str = input_d.get('answer', '')
-        ans = self.extract_answer(ans).strip()
-        if not ans:
-            logger.error(f'No ground truth answer found in the input: {input_d}')
-        return ans
-
-    def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = 'checkpoint') -> str:
-        """
-        Parse the model output to get the answer. Could be the best choice index.
-
-        Args:
-            result: Predicted answer from the model. Usually a string for chat.
-            raw_input_d (dict): The raw input. Depending on the dataset.
-            eval_type (str): 'checkpoint' or 'service'
-
-        Returns:
-            The parsed answer. Depending on the dataset. Usually a string for chat.
-        """
-        # Note: to use same extraction method for both of checkpoint and custom.
-        return self.extract_answer(result)
-
-    def match(self, gold: str, pred: str) -> float:
-        """
-        Match the gold answer and predicted answer.
-
-        Args:
-            gold (str): The golden answer. Note: to be extracted.
-            pred (str): The extracted prediction. Usually a string for chat/multiple-choice-questions.
-                        e.g. 'B'
-        """
-
-        def number_equal(gold_ans, pred_ans):
-            if pred_ans is None:
-                return False
-            try:
-                return math.isclose(eval(gold_ans), eval(pred_ans), rel_tol=0, abs_tol=1e-4)
-            except:
-                logger.warning(f'##report##Cannot compare two numbers: gold_ans={gold_ans}, pred_ans={pred_ans}')
-                return False
-
-        return number_equal(gold_ans=gold, pred_ans=pred)
-
-    @classmethod
-    def _generate_prompt(cls, use_fewshot: bool = True) -> str:
-        if use_fewshot:
-            # Use 4-shot examples by system
-            context = (
-                "Question: Angelo and Melanie want to plan how many hours over the next week they should study together for their test next week. They have 2 chapters of their textbook to study and 4 worksheets to memorize. They figure out that they should dedicate 3 hours to each chapter of their textbook and 1.5 hours for each worksheet. If they plan to study no more than 4 hours each day, how many days should they plan to study total over the next week if they take a 10-minute break every hour, include 3 10-minute snack breaks each day, and 30 minutes for lunch each day?\nLet's think step by step\n"
-                'Angelo and Melanie think they should dedicate 3 hours to each of the 2 chapters, 3 hours x 2 chapters = 6 hours total.\nFor the worksheets they plan to dedicate 1.5 hours for each worksheet, 1.5 hours x 4 worksheets = 6 hours total.\nAngelo and Melanie need to start with planning 12 hours to study, at 4 hours a day, 12 / 4 = 3 days.\nHowever, they need to include time for breaks and lunch. Every hour they want to include a 10-minute break, so 12 total hours x 10 minutes = 120 extra minutes for breaks.\nThey also want to include 3 10-minute snack breaks, 3 x 10 minutes = 30 minutes.\nAnd they want to include 30 minutes for lunch each day, so 120 minutes for breaks + 30 minutes for snack breaks + 30 minutes for lunch = 180 minutes, or 180 / 60 minutes per hour = 3 extra hours.\nSo Angelo and Melanie want to plan 12 hours to study + 3 hours of breaks = 15 hours total.\nThey want to study no more than 4 hours each day, 15 hours / 4 hours each day = 3.75\nThey will need to plan to study 4 days to allow for all the time they need.\nThe answer is 4\n\n'
-                "Question: Mark's basketball team scores 25 2 pointers, 8 3 pointers and 10 free throws.  Their opponents score double the 2 pointers but half the 3 pointers and free throws.  What's the total number of points scored by both teams added together?\nLet's think step by step\n"
-                "Mark's team scores 25 2 pointers, meaning they scored 25*2= 50 points in 2 pointers.\nHis team also scores 6 3 pointers, meaning they scored 8*3= 24 points in 3 pointers\nThey scored 10 free throws, and free throws count as one point so they scored 10*1=10 points in free throws.\nAll together his team scored 50+24+10= 84 points\nMark's opponents scored double his team's number of 2 pointers, meaning they scored 50*2=100 points in 2 pointers.\nHis opponents scored half his team's number of 3 pointers, meaning they scored 24/2= 12 points in 3 pointers.\nThey also scored half Mark's team's points in free throws, meaning they scored 10/2=5 points in free throws.\nAll together Mark's opponents scored 100+12+5=117 points\nThe total score for the game is both team's scores added together, so it is 84+117=201 points\nThe answer is 201\n\n"
-                "Question: Bella has two times as many marbles as frisbees. She also has 20 more frisbees than deck cards. If she buys 2/5 times more of each item, what would be the total number of the items she will have if she currently has 60 marbles?\nLet's think step by step\n"
-                "When Bella buys 2/5 times more marbles, she'll have increased the number of marbles by 2/5*60 = 24\nThe total number of marbles she'll have is 60+24 = 84\nIf Bella currently has 60 marbles, and she has two times as many marbles as frisbees, she has 60/2 = 30 frisbees.\nIf Bella buys 2/5 times more frisbees, she'll have 2/5*30 = 12 more frisbees.\nThe total number of frisbees she'll have will increase to 30+12 = 42\nBella also has 20 more frisbees than deck cards, meaning she has 30-20 = 10 deck cards\nIf she buys 2/5 times more deck cards, she'll have 2/5*10 = 4 more deck cards.\nThe total number of deck cards she'll have is 10+4 = 14\nTogether, Bella will have a total of 14+42+84 = 140 items\nThe answer is 140\n\n"
-                "Question: A group of 4 fruit baskets contains 9 apples, 15 oranges, and 14 bananas in the first three baskets and 2 less of each fruit in the fourth basket. How many fruits are there?\nLet's think step by step\n"
-                'For the first three baskets, the number of apples and oranges in one basket is 9+15=24\nIn total, together with bananas, the number of fruits in one basket is 24+14=38 for the first three baskets.\nSince there are three baskets each having 38 fruits, there are 3*38=114 fruits in the first three baskets.\nThe number of apples in the fourth basket is 9-2=7\nThere are also 15-2=13 oranges in the fourth basket\nThe combined number of oranges and apples in the fourth basket is 13+7=20\nThe fourth basket also contains 14-2=12 bananas.\nIn total, the fourth basket has 20+12=32 fruits.\nThe four baskets together have 32+114=146 fruits.\nThe answer is 146\n\n'
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        DELIM = '####'
+        question = record['question']
+        answer = record['answer'].split(DELIM)
+        target = answer.pop().strip()
+        reasoning = DELIM.join(answer)
+
+        return Sample(input=question, target=target, metadata={'reasoning': reasoning.strip()})
+
+    def sample_to_fewshot(self, sample: Sample) -> str:
+        if sample.metadata:
+            return (
+                f'{sample.input}\n\nReasoning:\n' + f"{sample.metadata['reasoning']}\n\n" + f'ANSWER: {sample.target}'
             )
         else:
-            context = ''
-        return context
-
-    @staticmethod
-    def extract_answer(s: str) -> str:
-        _PAT_LAST_DIGIT = re.compile(r'([+-])?(?=([0-9]|\.[0-9]))(0|([1-9](\d{0,2}(,\d{3})*)|\d*))?(\.\d*)?(?=\D|$)')
-        match = list(_PAT_LAST_DIGIT.finditer(s))
-        if match:
-            last_digit = match[-1].group().replace(',', '').replace('+', '').strip().strip('.')
-            # print(f"The last digit in {s} is {last_digit}")
-        else:
-            last_digit = None
-            print(f'No digits found in {s!r}', flush=True)
+            return ''
+
+    def extract_answer(self, prediction: str, task_state: TaskState):
+        from evalscope.filters.extraction import RegexFilter
 
-        return last_digit
+        regex = RegexFilter(regex_pattern=r'(-?[0-9.,]{2,})|(-?[0-9]+)', group_select=-1)
+        res = regex(prediction)
+        return res.replace(',', '').replace('+', '').strip().strip('.')

evalscope/benchmarks/hellaswag/hellaswag_adapter.py

@@ -3,118 +3,56 @@ import numpy as np
 import os
 import re
 
-from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.constants import EvalType, OutputType
-from evalscope.metrics import exact_match
-from evalscope.metrics.completion_parsers import ResponseParser
-from evalscope.utils.io_utils import jsonl_to_list
+from evalscope.api.benchmark import BenchmarkMeta, MultiChoiceAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
 from evalscope.utils.logger import get_logger
+from evalscope.utils.multi_choices import MultipleChoiceTemplate
 
 # flake8: noqa
 
 logger = get_logger()
 
 
-@Benchmark.register(
-    name='hellaswag',
-    pretty_name='HellaSwag',
-    tags=['Commonsense', 'MCQ', 'Knowledge'],
-    description=
-    'HellaSwag is a benchmark for commonsense reasoning in natural language understanding tasks. It consists of multiple-choice questions where the model must select the most plausible continuation of a given context.',
-    dataset_id='modelscope/hellaswag',
-    model_adapter=OutputType.MULTIPLE_CHOICE,
-    output_types=[OutputType.MULTIPLE_CHOICE, OutputType.GENERATION],
-    subset_list=['default'],
-    metric_list=['AverageAccuracy'],
-    few_shot_num=0,
-    train_split='train',
-    eval_split='validation',
-    prompt_template='{query}',  # noqa: E501
+@register_benchmark(
+    BenchmarkMeta(
+        name='hellaswag',
+        pretty_name='HellaSwag',
+        tags=[Tags.COMMONSENSE, Tags.MULTIPLE_CHOICE, Tags.KNOWLEDGE],
+        description=
+        'HellaSwag is a benchmark for commonsense reasoning in natural language understanding tasks. It consists of multiple-choice questions where the model must select the most plausible continuation of a given context.',
+        dataset_id='evalscope/hellaswag',
+        metric_list=['acc'],
+        subset_list=['default'],
+        few_shot_num=0,
+        train_split=None,
+        eval_split='validation',
+        prompt_template=MultipleChoiceTemplate.SINGLE_ANSWER,
+    )
 )
-class HellaSwagAdapter(DataAdapter):
+class HellaSwagAdapter(MultiChoiceAdapter):
 
     def __init__(self, **kwargs):
-
-        few_shot_num = kwargs.get('few_shot_num', 0)
-        if few_shot_num != 0:
-            logger.warning(f'few_shot_num should be 0 for HellaSwag, but got {few_shot_num}. Use 0-shot by default.')
-            kwargs['few_shot_num'] = 0
-
         super().__init__(**kwargs)
-        self.choices = ['A', 'B', 'C', 'D']
-
-    def load_from_disk(self, dataset_name_or_path, subset_list, work_dir, **kwargs) -> dict:
-        data_dict = {}
-        for subset_name in subset_list:
-            data_dict[subset_name] = {}
-            for split in [self.train_split, self.eval_split]:
-                if os.path.exists(dataset_name_or_path):
-                    file_path = os.path.join(dataset_name_or_path, f'hellaswag_{split}.jsonl')
-                else:
-                    file_path = os.path.join(work_dir, dataset_name_or_path, f'hellaswag_{split}.jsonl')
-                if os.path.exists(file_path):
-                    data_dict[subset_name][split] = jsonl_to_list(file_path)
-
-        return data_dict
-
-    def gen_prompt(self, input_d: dict, few_shot_list: list, **kwargs) -> dict:
-        """
-        Generate model prompt from raw data, unify the prompt format for HellaSwag benchmark.
-
-        Args:
-            input_d (dict): The raw input. A single data format of the HellaSwag:
-
-            {
-                'ind': 4,
-                'activity_label': 'Removing ice from car',
-                'ctx_a': 'Then, the man writes over the snow covering the window of a car, and a woman wearing winter clothes smiles.',
-                'ctx_b': 'then',
-                'ctx': 'Then, the man writes over the snow covering the window of a car, and a woman wearing winter clothes smiles. then',
-                'endings': [', the man adds wax to the windshield and cuts it.', ', a person board a ski lift, while two men supporting the head of the person wearing winter clothes snow as the we girls sled.', ', the man puts on a christmas coat, knitted with netting.', ', the man continues removing the snow on his car.'],
-                'source_id': 'activitynet~v_-1IBHYS3L-Y',
-                'split': 'train',
-                'split_type': 'indomain',
-                'label': '3'
-            }
-
-        Returns:
-            Refer to function: evalscope.benchmarks.data_adapter.DataAdapter.gen_prompt for details.
-        """
-
-        endings: list = [self._preprocess(ending) for ending in input_d['endings']]
-
-        few_shot_prompts = [
-            self._generate_prompt(input_d=sample, endings=endings, include_answer=True) for sample in few_shot_list
-        ]
-        context: str = '\n'.join(few_shot_prompts) + '\n'
-        query = context.strip() + self._generate_prompt(input_d=input_d, endings=endings, include_answer=False)
 
-        full_prompt = self.prompt_template.format(query=query)
-        return self.gen_prompt_data(full_prompt)
+    def record_to_sample(self, record) -> Sample:
+        # Preprocess endings
+        endings = [self._preprocess(ending) for ending in record['endings']]
 
-    def get_gold_answer(self, input_d: dict) -> str:
-        # Get the gold choice from the label
-        return self.choices[int(input_d['label'])]
+        # Create context
+        ctx = record['ctx_a'] + ' ' + record['ctx_b'].capitalize()
+        context = self._preprocess(ctx)
 
-    def parse_pred_result(self, result: list, raw_input_d: dict = None, eval_type: str = EvalType.CHECKPOINT) -> str:
-        """
-        Parse the model output to get the answer. Could be the best choice index.
+        # Get target choice letter
+        target_letter = ['A', 'B', 'C', 'D'][int(record['label'])]
 
-        Args:
-            result: Predicted answer from the model. Usually a string for chat.
-            raw_input_d: The raw input dict.
-            eval_type: The evaluation type. e.g. checkpoint, service, custom.
-
-        Returns:
-            The parsed answer. Depending on the dataset. Usually a string for chat.
-        """
-        if self.model_adapter == OutputType.MULTIPLE_CHOICE:
-            return result
-        else:
-            return ResponseParser.parse_first_option(result, options=self.choices)
-
-    def match(self, gold: str, pred: str) -> float:
-        return exact_match(gold=str(gold), pred=str(pred))
+        return Sample(
+            input=context,
+            choices=endings,
+            target=target_letter,
+            metadata={'activity_label': record.get('activity_label', 'unknown')},
+        )
 
     def _preprocess(self, text):
         text = text.strip()
@@ -122,30 +60,3 @@ class HellaSwagAdapter(DataAdapter):
         text = re.sub('\\[.*?\\]', '', text)
         text = text.replace('  ', ' ')
         return text
-
-    def _generate_prompt(self, input_d: dict, endings: list, include_answer=True) -> str:
-        """
-        Generate prompt for HellaSwag dataset.
-
-        Args:
-            input_d: a single data of the hellaswag.
-            endings:  preprocessed endings
-            include_answer: bool
-
-        Returns:
-
-        """
-
-        ctx = input_d['ctx_a'] + ' ' + input_d['ctx_b'].capitalize()
-        # example: str = cls._preprocess(input_d['activity_label'] + ': ' + ctx)
-        example: str = self._preprocess(ctx)
-
-        example += '\nQuestion: Which ending makes the most sense?'
-        for i, ending in enumerate(endings):
-            example += f'\n{self.choices[i]}. {ending}'
-        example += '\nYou may choose from A, B, C, D. Derive your final answer as `The answer is ...`.'
-
-        if include_answer:
-            example += '{}\n\n'.format(endings[int(input_d['label'])])
-
-        return example

evalscope/benchmarks/hle/hle_adapter.py

@@ -1,9 +1,13 @@
 import re
-from collections import defaultdict
-from typing import Any, List
-
-from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.metrics import DEFAULT_PROMPT_TEMPLATE, LLMJudge, exact_match, mean
+from typing import Any, Dict, List
+
+from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.evaluator import TaskState
+from evalscope.api.messages import ChatMessage, ChatMessageSystem, ChatMessageUser, Content, ContentImage, ContentText
+from evalscope.api.metric import Score
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
 from evalscope.utils.logger import get_logger
 
 # flake8: noqa
@@ -21,98 +25,128 @@ SUBSET_LIST = [
     'Other',
 ]
 
+ANSWER_TYPE_EXACT_MATCH = 'exactMatch'
+ANSWER_TYPE_MULTIPLE_CHOICE = 'multipleChoice'
+
+# System prompt constants
+SYSTEM_EXACT_ANSWER = 'Your response should be in the following format:\nExplanation: {your explanation for your final answer}\nExact Answer: {your succinct, final answer}\nConfidence: {your confidence score between 0% and 100% for your answer}'
+
+SYSTEM_MC = 'Your response should be in the following format:\nExplanation: {your explanation for your answer choice}\nAnswer: {your chosen answer}\nConfidence: {your confidence score between 0% and 100% for your answer}'
+
+JUDGE_PROMPT = """Judge whether the following [response] to [question] is correct or not based on the precise and unambiguous [correct_answer] below.
+
+[question]: {question}
+
+[response]: {response}
+
+[correct_answer]: {correct_answer}
 
-@Benchmark.register(
-    name='hle',
-    pretty_name="Humanity's-Last-Exam",
-    tags=['Knowledge', 'QA'],
-    description=
-    'Humanity\'s Last Exam (HLE) is a language model benchmark consisting of 2,500 questions across a broad range of subjects. It was created jointly by the Center for AI Safety and Scale AI. The benchmark classifies the questions into the following broad subjects: mathematics (41%), physics (9%), biology/medicine (11%), humanities/social science (9%), computer science/artificial intelligence (10%), engineering (4%), chemistry (7%), and other (9%). Around 14% of the questions require the ability to understand both text and images, i.e., multi-modality. 24% of the questions are multiple-choice; the rest are short-answer, exact-match questions.',  # noqa: E501
-    dataset_id='cais/hle',
-    subset_list=SUBSET_LIST,
-    metric_list=['AverageAccuracy'],
-    few_shot_num=0,
-    train_split=None,
-    eval_split='test',
-    prompt_template='{query}\n\nPlease reason step by step, and put your final answer within \\boxed{{}}.',
+Your judgment must focus only on if there are meaningful differences between [correct_answer] and the [response]. Do not comment on any background to the problem, do not attempt to solve the problem, do not argue for any answer different than [correct_answer], focus only on whether the answers match. Explain why the [response] is correct or incorrect based on [correct_answer] in one or two sentences. Finally, write your answer in the format 'GRADE: C' for correct answer or 'GRADE: I' for incorrect answer.
+"""
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='hle',
+        pretty_name="Humanity's-Last-Exam",
+        tags=[Tags.KNOWLEDGE, Tags.QA],
+        description='Humanity\'s Last Exam (HLE) is a language model benchmark consisting of 2,500 '
+        'questions across a broad range of subjects. It was created jointly by the Center '
+        'for AI Safety and Scale AI. The benchmark classifies the questions into the '
+        'following broad subjects: mathematics (41%), physics (9%), biology/medicine (11%), '
+        'humanities/social science (9%), computer science/artificial intelligence (10%), '
+        'engineering (4%), chemistry (7%), and other (9%). Around 14% of the questions '
+        'require the ability to understand both text and images, i.e., multi-modality. '
+        '24% of the questions are multiple-choice; the rest are short-answer, exact-match questions. '
+        'To evaluate the performance of model without multi-modality capabilities, please set the extra_params["include_multi_modal"] to False.',  # noqa: E501
+        dataset_id='cais/hle',
+        subset_list=SUBSET_LIST,
+        metric_list=['acc'],
+        eval_split='test',
+        prompt_template='{question}',
+        extra_params={'include_multi_modal': True}
+    )
 )
-class HLEAdapter(DataAdapter):
+class HLEAdapter(DefaultDataAdapter):
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
 
-        self.llm_as_a_judge = True
-
-    def load(self, **kwargs):
-        kwargs['subset_list'] = ['default']
-        data_dict = super().load(**kwargs)
-        return self.reformat_subset(data_dict, subset_key='category', format='{}')
-
-    def gen_prompt(self, input_d: dict, subset_name: str, few_shot_list: list, **kwargs) -> dict:
-        # remove image preview
-        input_d.pop('image_preview', None)
-        input_d.pop('rationale_image', None)
-        # generate prompt
-        question = input_d['question']
-        prompt = self.prompt_template.format(query=question)
-        image = input_d.get('image', None)
-        # build messages for multi-modal input
-        messages = []
-        if self.system_prompt:
-            messages.append({'role': 'system', 'content': self.system_prompt})
-        if image:
-            messages.append({
-                'role':
-                'user',
-                'content': [{
-                    'type': 'text',
-                    'text': prompt
-                }, {
-                    'type': 'image_url',
-                    'image_url': {
-                        'url': image
-                    }
-                }]
-            })
-        else:
-            messages.append({'role': 'user', 'content': prompt})
-        return self.gen_prompt_data(prompt='', messages=messages)
-
-    def get_gold_answer(self, input_d: dict) -> str:
-        return input_d['answer']
-
-    def parse_pred_result(self, result: str, raw_input_d: dict = None, **kwargs) -> str:
-        # Extract the answer from the model output \boxed{answer}
-        match = re.search(r'\\boxed{([^}]*)}', result)
-        if match:
-            return match.group(1).strip()
-        else:
-            logger.warning(f'No answer found in the model output: {result}')
-            return ''
-
-    def llm_parse_pred_result(self, result, raw_input_d=None, **kwargs) -> str:
-        return result.strip()
-
-    def match(self, gold: str, pred: str) -> dict:
-        # simple match
-        return {
-            'AverageAccuracy': 1.0 if exact_match(gold, pred) else 0.0,
-        }
-
-    def llm_match(self, gold: Any, pred: Any, judge: LLMJudge, **kwargs) -> dict:
-        raw_input = kwargs.get('raw_input', None)
-        question = raw_input['question']
-        # get grading response
-        prompt = judge.build_prompt(pred, gold, question)
-        judge_response = judge(prompt)
-        score = judge.get_score(judge_response)
-        return {
-            'AverageAccuracy': score,
-            'response': judge_response,
+        self._use_llm_judge = True  # Use LLM as a judge by default
+        self.reformat_subset = True
+        self.include_multi_modal = self.extra_params.get('include_multi_modal', True)
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        answer_type = record['answer_type']
+        system_prompt = (SYSTEM_EXACT_ANSWER if answer_type == ANSWER_TYPE_EXACT_MATCH else SYSTEM_MC)
+        text_content = ContentText(text=record['question'])
+
+        content: List[Content] = [text_content]
+        if record['image']:
+            image_content = ContentImage(image=record['image'])
+            content.append(image_content)
+
+        messages: List[ChatMessage] = [
+            ChatMessageSystem(content=system_prompt),
+            ChatMessageUser(content=content),
+        ]
+        return Sample(
+            input=messages,
+            subset_key=record['category'],
+            metadata={
+                'uid': record['id'],
+                'author_name': record['author_name'],
+                'rationale': record['rationale'],
+                'raw_subject': record['raw_subject'],
+                'category': record['category'],
+                'has_image': bool(record['image']),
+            },
+            target=record['answer'],
+        )
+
+    def sample_filter(self, sample):
+        if not self.include_multi_modal:
+            if sample.metadata is not None and sample.metadata['has_image']:
+                return False
+        return True
+
+    def llm_match_score(
+        self,
+        original_prediction: str,
+        filtered_prediction: str,
+        reference: str,
+        task_state: TaskState,
+    ) -> Score:
+        score = Score(
+            extracted_prediction=filtered_prediction,
+            prediction=original_prediction,
+        )
+
+        confidence = 100
+        if task_state.output and task_state.output.completion:
+            confidence_match = re.search(r'confidence:\s*(\d+)', task_state.output.completion, re.IGNORECASE)
+            if confidence_match:
+                confidence = int(confidence_match.group(1))
+
+        judge_prompt = JUDGE_PROMPT.format(
+            question=task_state.input_text, response=filtered_prediction, correct_answer=reference
+        )
+
+        # Request judge and obtain score
+        judge_response = self.llm_judge.judge(prompt=judge_prompt)
+
+        # Parse judge response to get accuracy score
+        accuracy_score = re.search(r'GRADE:\s*([CI])', judge_response, re.IGNORECASE)
+        if accuracy_score:
+            score.value = {
+                'acc': 1.0 if accuracy_score.group(1) == 'C' else 0.0,
+            }
+        score.explanation = f'LLM judge: {judge_response}'
+        score.metadata = {
+            'source': 'llm_judge',
+            'judge_strategy': self.judge_strategy,
+            'model': self.llm_judge.model_id,
+            'confidence': confidence,
         }
-
-    def compute_metric(self, review_res_list: List[dict], **kwargs) -> List[dict]:
-        # zip dict answers
-        res_dict = super().compute_dict_metric(review_res_list, **kwargs)
-
-        return super().compute_metric(res_dict, **kwargs)
+        score.main_score_name = 'acc'
+        return score