evalscope 0.17.1__py3-none-any.whl → 1.0.1__py3-none-any.whl

evalscope/benchmarks/hle/hle_adapter.py

@@ -1,9 +1,13 @@
 import re
-from collections import defaultdict
-from typing import Any, List
-
-from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.metrics import DEFAULT_PROMPT_TEMPLATE, LLMJudge, exact_match, mean
+from typing import Any, Dict, List
+
+from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.evaluator import TaskState
+from evalscope.api.messages import ChatMessage, ChatMessageSystem, ChatMessageUser, Content, ContentImage, ContentText
+from evalscope.api.metric import Score
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
 from evalscope.utils.logger import get_logger
 
 # flake8: noqa
@@ -21,98 +25,128 @@ SUBSET_LIST = [
     'Other',
 ]
 
+ANSWER_TYPE_EXACT_MATCH = 'exactMatch'
+ANSWER_TYPE_MULTIPLE_CHOICE = 'multipleChoice'
+
+# System prompt constants
+SYSTEM_EXACT_ANSWER = 'Your response should be in the following format:\nExplanation: {your explanation for your final answer}\nExact Answer: {your succinct, final answer}\nConfidence: {your confidence score between 0% and 100% for your answer}'
+
+SYSTEM_MC = 'Your response should be in the following format:\nExplanation: {your explanation for your answer choice}\nAnswer: {your chosen answer}\nConfidence: {your confidence score between 0% and 100% for your answer}'
+
+JUDGE_PROMPT = """Judge whether the following [response] to [question] is correct or not based on the precise and unambiguous [correct_answer] below.
+
+[question]: {question}
+
+[response]: {response}
+
+[correct_answer]: {correct_answer}
 
-@Benchmark.register(
-    name='hle',
-    pretty_name="Humanity's-Last-Exam",
-    tags=['Knowledge', 'QA'],
-    description=
-    'Humanity\'s Last Exam (HLE) is a language model benchmark consisting of 2,500 questions across a broad range of subjects. It was created jointly by the Center for AI Safety and Scale AI. The benchmark classifies the questions into the following broad subjects: mathematics (41%), physics (9%), biology/medicine (11%), humanities/social science (9%), computer science/artificial intelligence (10%), engineering (4%), chemistry (7%), and other (9%). Around 14% of the questions require the ability to understand both text and images, i.e., multi-modality. 24% of the questions are multiple-choice; the rest are short-answer, exact-match questions.',  # noqa: E501
-    dataset_id='cais/hle',
-    subset_list=SUBSET_LIST,
-    metric_list=['AverageAccuracy'],
-    few_shot_num=0,
-    train_split=None,
-    eval_split='test',
-    prompt_template='{query}\n\nPlease reason step by step, and put your final answer within \\boxed{{}}.',
+Your judgment must focus only on if there are meaningful differences between [correct_answer] and the [response]. Do not comment on any background to the problem, do not attempt to solve the problem, do not argue for any answer different than [correct_answer], focus only on whether the answers match. Explain why the [response] is correct or incorrect based on [correct_answer] in one or two sentences. Finally, write your answer in the format 'GRADE: C' for correct answer or 'GRADE: I' for incorrect answer.
+"""
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='hle',
+        pretty_name="Humanity's-Last-Exam",
+        tags=[Tags.KNOWLEDGE, Tags.QA],
+        description='Humanity\'s Last Exam (HLE) is a language model benchmark consisting of 2,500 '
+        'questions across a broad range of subjects. It was created jointly by the Center '
+        'for AI Safety and Scale AI. The benchmark classifies the questions into the '
+        'following broad subjects: mathematics (41%), physics (9%), biology/medicine (11%), '
+        'humanities/social science (9%), computer science/artificial intelligence (10%), '
+        'engineering (4%), chemistry (7%), and other (9%). Around 14% of the questions '
+        'require the ability to understand both text and images, i.e., multi-modality. '
+        '24% of the questions are multiple-choice; the rest are short-answer, exact-match questions. '
+        'To evaluate the performance of model without multi-modality capabilities, please set the extra_params["include_multi_modal"] to False.',  # noqa: E501
+        dataset_id='cais/hle',
+        subset_list=SUBSET_LIST,
+        metric_list=['acc'],
+        eval_split='test',
+        prompt_template='{question}',
+        extra_params={'include_multi_modal': True}
+    )
 )
-class HLEAdapter(DataAdapter):
+class HLEAdapter(DefaultDataAdapter):
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
 
-        self.llm_as_a_judge = True
-
-    def load(self, **kwargs):
-        kwargs['subset_list'] = ['default']
-        data_dict = super().load(**kwargs)
-        return self.reformat_subset(data_dict, subset_key='category', format='{}')
-
-    def gen_prompt(self, input_d: dict, subset_name: str, few_shot_list: list, **kwargs) -> dict:
-        # remove image preview
-        input_d.pop('image_preview', None)
-        input_d.pop('rationale_image', None)
-        # generate prompt
-        question = input_d['question']
-        prompt = self.prompt_template.format(query=question)
-        image = input_d.get('image', None)
-        # build messages for multi-modal input
-        messages = []
-        if self.system_prompt:
-            messages.append({'role': 'system', 'content': self.system_prompt})
-        if image:
-            messages.append({
-                'role':
-                'user',
-                'content': [{
-                    'type': 'text',
-                    'text': prompt
-                }, {
-                    'type': 'image_url',
-                    'image_url': {
-                        'url': image
-                    }
-                }]
-            })
-        else:
-            messages.append({'role': 'user', 'content': prompt})
-        return self.gen_prompt_data(prompt='', messages=messages)
-
-    def get_gold_answer(self, input_d: dict) -> str:
-        return input_d['answer']
-
-    def parse_pred_result(self, result: str, raw_input_d: dict = None, **kwargs) -> str:
-        # Extract the answer from the model output \boxed{answer}
-        match = re.search(r'\\boxed{([^}]*)}', result)
-        if match:
-            return match.group(1).strip()
-        else:
-            logger.warning(f'No answer found in the model output: {result}')
-            return ''
-
-    def llm_parse_pred_result(self, result, raw_input_d=None, **kwargs) -> str:
-        return result.strip()
-
-    def match(self, gold: str, pred: str) -> dict:
-        # simple match
-        return {
-            'AverageAccuracy': 1.0 if exact_match(gold, pred) else 0.0,
-        }
-
-    def llm_match(self, gold: Any, pred: Any, judge: LLMJudge, **kwargs) -> dict:
-        raw_input = kwargs.get('raw_input', None)
-        question = raw_input['question']
-        # get grading response
-        prompt = judge.build_prompt(pred, gold, question)
-        judge_response = judge(prompt)
-        score = judge.get_score(judge_response)
-        return {
-            'AverageAccuracy': score,
-            'response': judge_response,
+        self._use_llm_judge = True  # Use LLM as a judge by default
+        self.reformat_subset = True
+        self.include_multi_modal = self.extra_params.get('include_multi_modal', True)
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        answer_type = record['answer_type']
+        system_prompt = (SYSTEM_EXACT_ANSWER if answer_type == ANSWER_TYPE_EXACT_MATCH else SYSTEM_MC)
+        text_content = ContentText(text=record['question'])
+
+        content: List[Content] = [text_content]
+        if record['image']:
+            image_content = ContentImage(image=record['image'])
+            content.append(image_content)
+
+        messages: List[ChatMessage] = [
+            ChatMessageSystem(content=system_prompt),
+            ChatMessageUser(content=content),
+        ]
+        return Sample(
+            input=messages,
+            subset_key=record['category'],
+            metadata={
+                'uid': record['id'],
+                'author_name': record['author_name'],
+                'rationale': record['rationale'],
+                'raw_subject': record['raw_subject'],
+                'category': record['category'],
+                'has_image': bool(record['image']),
+            },
+            target=record['answer'],
+        )
+
+    def sample_filter(self, sample):
+        if not self.include_multi_modal:
+            if sample.metadata is not None and sample.metadata['has_image']:
+                return False
+        return True
+
+    def llm_match_score(
+        self,
+        original_prediction: str,
+        filtered_prediction: str,
+        reference: str,
+        task_state: TaskState,
+    ) -> Score:
+        score = Score(
+            extracted_prediction=filtered_prediction,
+            prediction=original_prediction,
+        )
+
+        confidence = 100
+        if task_state.output and task_state.output.completion:
+            confidence_match = re.search(r'confidence:\s*(\d+)', task_state.output.completion, re.IGNORECASE)
+            if confidence_match:
+                confidence = int(confidence_match.group(1))
+
+        judge_prompt = JUDGE_PROMPT.format(
+            question=task_state.input_text, response=filtered_prediction, correct_answer=reference
+        )
+
+        # Request judge and obtain score
+        judge_response = self.llm_judge.judge(prompt=judge_prompt)
+
+        # Parse judge response to get accuracy score
+        accuracy_score = re.search(r'GRADE:\s*([CI])', judge_response, re.IGNORECASE)
+        if accuracy_score:
+            score.value = {
+                'acc': 1.0 if accuracy_score.group(1) == 'C' else 0.0,
+            }
+        score.explanation = f'LLM judge: {judge_response}'
+        score.metadata = {
+            'source': 'llm_judge',
+            'judge_strategy': self.judge_strategy,
+            'model': self.llm_judge.model_id,
+            'confidence': confidence,
         }
-
-    def compute_metric(self, review_res_list: List[dict], **kwargs) -> List[dict]:
-        # zip dict answers
-        res_dict = super().compute_dict_metric(review_res_list, **kwargs)
-
-        return super().compute_metric(res_dict, **kwargs)
+        score.main_score_name = 'acc'
+        return score

evalscope/benchmarks/humaneval/humaneval_adapter.py

@@ -1,7 +1,15 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
+# flake8: noqa: E501
 import re
-
-from evalscope.benchmarks import Benchmark, DataAdapter
+from typing import Any, Dict
+
+from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.evaluator import TaskState
+from evalscope.api.messages.chat_message import ChatMessageUser
+from evalscope.api.metric import Score
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
 from evalscope.utils.logger import get_logger
 
 logger = get_logger()
@@ -10,28 +18,28 @@ logger = get_logger()
 # {"task_id": "HumanEval/0", "prompt": "from typing import List\n\n\ndef has_close_elements(numbers: List[float], threshold: float) -> bool:\n    \"\"\" Check if in given list of numbers, are any two numbers closer to each other than\n    given threshold.\n    >>> has_close_elements([1.0, 2.0, 3.0], 0.5)\n    False\n    >>> has_close_elements([1.0, 2.8, 3.0, 4.0, 5.0, 2.0], 0.3)\n    True\n    \"\"\"\n", "entry_point": "has_close_elements", "canonical_solution": "    for idx, elem in enumerate(numbers):\n        for idx2, elem2 in enumerate(numbers):\n            if idx != idx2:\n                distance = abs(elem - elem2)\n                if distance < threshold:\n                    return True\n\n    return False\n", "test": "\n\nMETADATA = {\n    'author': 'jt',\n    'dataset': 'test'\n}\n\n\ndef check(candidate):\n    assert candidate([1.0, 2.0, 3.9, 4.0, 5.0, 2.2], 0.3) == True\n    assert candidate([1.0, 2.0, 3.9, 4.0, 5.0, 2.2], 0.05) == False\n    assert candidate([1.0, 2.0, 5.9, 4.0, 5.0], 0.95) == True\n    assert candidate([1.0, 2.0, 5.9, 4.0, 5.0], 0.8) == False\n    assert candidate([1.0, 2.0, 3.0, 4.0, 5.0, 2.0], 0.1) == True\n    assert candidate([1.1, 2.2, 3.1, 4.1, 5.1], 1.0) == True\n    assert candidate([1.1, 2.2, 3.1, 4.1, 5.1], 0.5) == False\n\n"}  # noqa
 
 
-@Benchmark.register(
-    name='humaneval',
-    pretty_name='HumanEval',
-    tags=['Coding'],
-    description=
-    'HumanEval is a benchmark for evaluating the ability of code generation models to write Python functions based on given specifications. It consists of programming tasks with a defined input-output behavior.',  # noqa: E501
-    dataset_id='modelscope/humaneval',
-    subset_list=['openai_humaneval'],
-    metric_list=['Pass@1'],
-    few_shot_num=0,
-    train_split=None,
-    eval_split='test',
-    prompt_template=
-    'Read the following function signature and docstring, and fully implement the function described. Your response should only contain the code for this function.\n{query}',  # noqa: E501
-    extra_params={
-        'num_workers': 4,
-        'timeout': 4
-    },
+@register_benchmark(
+    BenchmarkMeta(
+        name='humaneval',
+        pretty_name='HumanEval',
+        tags=[Tags.CODING],
+        description=
+        'HumanEval is a benchmark for evaluating the ability of code generation models to write Python functions based on given specifications. It consists of programming tasks with a defined input-output behavior.',
+        dataset_id='opencompass/humaneval',
+        subset_list=['openai_humaneval'],
+        metric_list=['Pass@1'],
+        eval_split='test',
+        prompt_template=
+        'Read the following function signature and docstring, and fully implement the function described. Your response should only contain the code for this function.\n{question}',
+        extra_params={
+            'num_workers': 4,
+            'timeout': 4
+        },
+    )
 )
-class HumanevalAdapter(DataAdapter):
+class HumanevalAdapter(DefaultDataAdapter):
     """
-    A placeholder for humaneval adapter, see HumanevalEvaluator for implementation.
+    HumanEval adapter using the new data processing framework.
     """
 
     def __init__(self, **kwargs):
@@ -39,9 +47,11 @@ class HumanevalAdapter(DataAdapter):
             from human_eval.data import stream_jsonl, write_jsonl
             from human_eval.evaluation import check_correctness
         except ImportError:
-            raise ImportError('Please install human_eval:'
-                              'https://github.com/openai/human-eval/tree/master#installation , '
-                              'Note that you need to enable the execution code in the human_eval/execution.py first.')
+            raise ImportError(
+                'Please install human_eval:'
+                'https://github.com/openai/human-eval/tree/master#installation , '
+                'Note that you need to enable the execution code in the human_eval/execution.py first.'
+            )
         super().__init__(**kwargs)
 
         extra_params = kwargs.get('extra_params', {})
@@ -53,41 +63,62 @@ class HumanevalAdapter(DataAdapter):
         self.write_jsonl_func = write_jsonl
         self.eval_func = check_correctness
 
-    def load_from_disk(self, dataset_name_or_path, subset_list, work_dir, **kwargs) -> dict:
-        data_dict = {}
-        for subset_name in subset_list:
-            data_dict[subset_name] = {}
-            # [{'task_id': '', 'prompt': '', 'entry_point': '', 'canonical_solution': '', 'test': ''}, ...]
-            data_dict[subset_name][self.eval_split] = [task for task in self.read_problems_func(dataset_name_or_path)]
-
-        return data_dict
-
-    def gen_prompt(self, input_d: dict, few_shot_list: list, **kwargs) -> dict:
-        """
-        Generate prompt for the model.
-
-        Args:
-            input_d (dict): The raw input. A single data format of the Humaneval:
-            {'task_id': '', 'prompt': '', 'entry_point': '', 'canonical_solution': '', 'test': ''}
-        """
-        query = input_d['prompt']
-        full_prompt = self.prompt_template.format(query=query)
-
-        return self.gen_prompt_data(full_prompt)
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        """Convert a data record to a Sample object."""
+        query = record['prompt']
+        full_prompt = self.prompt_template.format(question=query)
+
+        return Sample(
+            input=[ChatMessageUser(content=full_prompt)],
+            target=record['canonical_solution'],
+            metadata={
+                'task_id': record['task_id'],
+                'entry_point': record['entry_point'],
+                'prompt': record['prompt'],
+                'test': record['test'],
+            }
+        )
+
+    def extract_answer(self, prediction: str, task_state: TaskState) -> str:
+        """Extract code from the prediction."""
+        return self._postprocess(prediction)
 
     @classmethod
     def _postprocess(cls, text: str) -> str:
+        """Extract code from markdown code blocks."""
         blocks = re.findall(r'```\w*\n(.*?)```', text, re.DOTALL)
         if len(blocks) >= 1:
             text = blocks[0]
         return text
 
-    def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = 'checkpoint') -> str:
-        return self._postprocess(result)
-
-    def get_gold_answer(self, input_d: dict) -> str:
-        return input_d
-
-    def match(self, gold: str, pred: str) -> float:
-        res = self.eval_func(gold, pred, self.timeout)
-        return float(res['passed'])
+    def match_score(
+        self, original_prediction: str, filtered_prediction: str, reference: str, task_state: TaskState
+    ) -> Score:
+        score = Score(
+            extracted_prediction=filtered_prediction,
+            prediction=original_prediction,
+        )
+
+        # Execute the code and check correctness
+        res = self.eval_func(task_state.metadata, filtered_prediction, self.timeout)
+        passed = res['passed']
+
+        score.value = {'pass': passed}
+        score.explanation = res.get('result', 'Code execution completed')
+        score.metadata = {'task_id': task_state.metadata['task_id'], 'timeout': self.timeout, 'execution_result': res}
+        score.main_score_name = 'pass'
+
+        return score
+
+    def aggregate_scores(self, sample_scores):
+        from evalscope.metrics.metric import PassAtK
+
+        # caculate pass@k here
+        agg_list = []
+        for metric in self.metric_list:
+            if metric.lower().startswith('pass@'):
+                k = int(metric.split('@')[1])
+                # Get the scores for this metric
+                agg = PassAtK(k)
+                agg_list.extend(agg(sample_scores))
+        return agg_list

evalscope/benchmarks/ifeval/ifeval_adapter.py

@@ -1,54 +1,83 @@
-from collections import defaultdict
 from typing import Any, Dict, List
 
-from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.constants import EvalType
-from evalscope.metrics import Metric, mean, metric_registry
-
-
-@Benchmark.register(
-    name='ifeval',
-    pretty_name='IFEval',
-    tags=['Instruction-Following'],
-    description=
-    'IFEval is a benchmark for evaluating instruction-following language models, focusing on their ability to understand and respond to various prompts. It includes a diverse set of tasks and metrics to assess model performance comprehensively.',  # noqa: E501
-    dataset_id='opencompass/ifeval',
-    subset_list=['default'],
-    metric_list=[
-        'prompt_level_strict_acc',
-        'inst_level_strict_acc',
-        'prompt_level_loose_acc',
-        'inst_level_loose_acc',
-    ],
-    few_shot_num=0,
-    train_split=None,
-    eval_split='train',
-    prompt_template='',
+from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.evaluator import TaskState
+from evalscope.api.messages import ChatMessageUser
+from evalscope.api.metric import Score
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='ifeval',
+        pretty_name='IFEval',
+        description=
+        'IFEval is a benchmark for evaluating instruction-following language models, focusing on their ability to understand and respond to various prompts. It includes a diverse set of tasks and metrics to assess model performance comprehensively.',  # noqa: E501
+        tags=[Tags.INSTRUCTION_FOLLOWING],
+        dataset_id='opencompass/ifeval',
+        subset_list=['default'],
+        metric_list=[
+            'prompt_level_strict',
+            'inst_level_strict',
+            'prompt_level_loose',
+            'inst_level_loose',
+        ],
+        few_shot_num=0,
+        train_split=None,
+        eval_split='train',
+        prompt_template='',
+    )
 )
-class IFEvalAdapter(DataAdapter):
+class IFEvalAdapter(DefaultDataAdapter):
 
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
 
-        # register metrics
-        metric_registry.register(Metric(name='prompt_level_strict_acc', object=mean))
-        metric_registry.register(Metric(name='inst_level_strict_acc', object=mean))
-        metric_registry.register(Metric(name='prompt_level_loose_acc', object=mean))
-        metric_registry.register(Metric(name='inst_level_loose_acc', object=mean))
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        """
+        Convert a data record to a Sample object.
 
-    def gen_prompt(self, input_d: dict, subset_name: str, few_shot_list: list, **kwargs) -> Any:
-        return self.gen_prompt_data(input_d['prompt'])
+        Args:
+            record (Dict[str, Any]): Input data record.
 
-    def get_gold_answer(self, input_d: dict) -> str:
-        return input_d
+        Returns:
+            Sample: Sample object with input, target, and metadata.
+        """
+        prompt = record.get('prompt', '')
+        message_list = [ChatMessageUser(content=prompt)]
 
-    def match(self, gold: Any, pred: Any) -> Dict:
+        return Sample(input=message_list, target='', metadata=record)
+
+    def match_score(
+        self, original_prediction: str, filtered_prediction: str, reference: Dict, task_state: TaskState
+    ) -> Score:
+        """
+        Calculate evaluation scores by comparing prediction with reference.
+        """
         from evalscope.benchmarks.ifeval.utils import process_results
 
-        return process_results(gold, [pred])
+        # Initialize the score object with prediction details
+        score = Score(
+            extracted_prediction=filtered_prediction,
+            prediction=original_prediction,
+        )
+
+        doc = task_state.metadata
+        try:
+            # Process results using the existing ifeval utility
+            results = process_results(doc, [filtered_prediction])
+            score.value.update(results)
+
+            # Set main score name
+            score.main_score_name = 'prompt_level_strict'
 
-    def compute_metric(self, review_res_list: List[dict], **kwargs) -> Any:
-        # aggregate review results
-        res_dict = super().compute_dict_metric(review_res_list, **kwargs)
+        except Exception as e:
+            logger.error(f'Error calculating ifeval metrics: {e}')
+            score.value = {}
 
-        return super().compute_metric(res_dict, **kwargs)
+        return score