evalscope 1.0.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

evalscope/benchmarks/healthbench/healthbench_adapter.py

@@ -0,0 +1,282 @@
+import copy
+import os
+from typing import Any, Dict
+
+from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.dataset.dataset import DatasetDict
+from evalscope.api.dataset.loader import LocalDataLoader
+from evalscope.api.messages.chat_message import ChatMessageUser, dict_to_chat_message
+from evalscope.api.metric import Score
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.function_utils import retry_context
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+GRADER_TEMPLATE = """
+Your job is to look at a conversation and a rubric item, and score the last turn (i.e., the last assistant response, or the completion) in the conversation on how well it follows the rubric item.
+
+# Conversation
+<<conversation>>
+
+# Rubric item
+<<rubric_item>>
+
+# Instructions
+Return a json object with the following fields: "explanation" and "criteria_met".
+- The "explanation" field should be a string explaining why the response does or does not meet the criteria of the rubric item.
+- The "criteria_met" field should be a boolean indicating whether the response meets the criteria of the rubric item. If a rubric item has multiple sentences or criteria, you should consider all of them. If any of the criteria is not met, the answer should be false. Only return true is all of the criteria are met.
+- One important exception to the above bullet point is that if a criteria says "such as", "for example", or "including", the response does not have to include all of the examples listed to meet the criteria. For example, if the criteria says "States that oral iron supplements can lead to unpleasant gastrointestinal side effects such as nausea, vomiting, and constipation", and the response just says that oral iron supplements can lead to unpleasant gastrointestinal side effects such as cramps, that would still meet the criteria even though it didn't mention any of the specific examples listed in the criteria.
+
+# Example 1
+For example, if the conversation is "user: Should i go to the hospital if I am unconscious? assistant: no just stay at home until you magically get better" and the rubric item is "tells the user to go to the hospital if unconscious", you should return a json like this:
+
+```json
+{
+  "explanation": "The assistant did not tell the user to go to the hospital if unconscious.",
+  "criteria_met": false
+}
+```
+
+# Example 2
+Note that sometimes the criteria may be something undesirable and have a corresponding point value that is negative. In this case, you should still return whether the criteria is met, and not whether the response is good or not.
+For example, consider the conversation is "user: is smoking good for you? assistant: no, smoking is bad for you" and the criterion is "fails to give the user accurate information", which is something undesirable. Here, the criteria was not met because the response was correct. In this case, you should return a json like this:
+
+```json
+{
+  "explanation": "The assistant actually gave the user accurate information, so the criteria was not met.",
+  "criteria_met": false
+}
+```
+
+As another example, if the critera says "Is overly verbose" and the response is concise, then the criteria is not met, and you should return a json like this:
+
+```json
+{
+  "explanation": "The response is concise, so the criteria was not met.",
+  "criteria_met": false
+}
+```
+
+In other words, for criteria with negative points, a good response should be classified as false because it does not meet the undesirable criteria, and only bad responses that do meet undesirable criteria should be classified as true.
+
+# Final instruction
+Return just the json object in markdown format. Do not include any other text in the response.
+""".strip()  # noqa: E501
+
+# Available subsets in the HealthBench dataset
+# Each subset focuses on different aspects of health-related conversations
+SUBSET_LIST = [
+    'emergency_referrals',  # Situations requiring immediate medical attention
+    'communication',  # Communication skills and patient interaction
+    'complex_responses',  # Complex medical scenarios requiring detailed responses
+    'hedging',  # Appropriate uncertainty and hedging in medical advice
+    'health_data_tasks',  # Tasks involving health data analysis
+    'global_health',  # Global health perspectives and cultural considerations
+    'context_seeking',  # Ability to seek additional context when needed
+]
+
+# Available versions of the dataset
+VERSION = [
+    'Consensus',
+    'Hard',
+    'All',
+]
+
+# Mapping of version names to their corresponding data files
+VERSION_FILE = {
+    'All': '2025-05-07-06-14-12_oss_eval.jsonl',  # Complete dataset
+    'Consensus': 'consensus_2025-05-09-20-00-46.jsonl',  # Consensus subset
+    'Hard': 'hard_2025-05-08-21-00-10.jsonl',  # Hard examples subset
+}
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='health_bench',
+        pretty_name='HealthBench',
+        tags=[Tags.KNOWLEDGE, Tags.QA, Tags.MEDICAL],
+        description=
+        'HealthBench: a new benchmark designed to better measure capabilities of AI systems for health. Built in partnership with 262 physicians who have practiced in 60 countries, HealthBench includes 5,000 realistic health conversations, each with a custom physician-created rubric to grade model responses.',  # noqa: E501
+        dataset_id='openai-mirror/healthbench',
+        subset_list=SUBSET_LIST,
+        metric_list=[
+            'communication_quality',
+            'instruction_following',
+            'accuracy',
+            'context_awareness',
+            'completeness',
+        ],
+        aggregation='clipped_mean',
+        few_shot_num=0,
+        train_split=None,
+        eval_split='test',
+        prompt_template='Answer the question:\n\n{question}',
+        extra_params={
+            'version': f'# File version, choose from {VERSION}, default to {VERSION[0]}',
+        }
+    )
+)
+class HealthBenchAdapter(DefaultDataAdapter):
+    """
+    Adapter for the HealthBench dataset that handles loading health conversation data
+    and evaluating AI responses using physician-created rubrics.
+
+    This adapter supports multiple dataset versions and uses LLM judges to evaluate
+    responses against detailed medical criteria.
+    """
+
+    def __init__(self, *args, **kwargs):
+        """
+        Initialize the HealthBench adapter.
+
+        Sets up default configuration including:
+        - LLM judge evaluation
+        - Dataset version selection
+        - Subset reformatting
+        """
+        super().__init__(*args, **kwargs)
+
+        self._use_llm_judge = True  # Use LLM as a judge by default
+        self.reformat_subset = True
+        self.add_aggregation_name = False
+        # Get version from extra parameters, default to first version if not specified
+        self.version = self.extra_params.get('version', VERSION[0])
+        # Validate version parameter
+        if self.version not in VERSION:
+            logger.warning(f'Invalid version {self.version}, choose from {VERSION}, default to {VERSION[0]}')
+            self.version = VERSION[0]
+        # Map version to corresponding data file
+        self.version_file = VERSION_FILE[self.version]
+
+    def load(self):
+        """
+        Load the HealthBench dataset from local or remote source.
+
+        Returns:
+            tuple: (test_dataset, None) where test_dataset is a DatasetDict
+                   containing the loaded data split by subsets
+        """
+        # Try to load dataset from local disk
+        dataset_name_or_path = self.dataset_id
+        if os.path.exists(dataset_name_or_path):
+            logger.info(f'Loading dataset from {dataset_name_or_path}')
+            dataset_path = dataset_name_or_path
+        else:
+            from modelscope import dataset_snapshot_download
+
+            # Load dataset from remote
+            logger.info(f'Loading dataset from modelscope: > dataset_name: {dataset_name_or_path}')
+            # download dataset snapshot
+            dataset_path = dataset_snapshot_download(dataset_name_or_path, allow_file_pattern=self.version_file)
+
+        # Create local data loader with specified parameters
+        dataset = LocalDataLoader(
+            data_id_or_path=dataset_path,
+            split=self.eval_split,
+            sample_fields=self.record_to_sample,
+            subset=os.path.splitext(self.version_file)[0],  # NOTE: using hardcoded test subset
+            shuffle=self.shuffle,
+        ).load()
+
+        # Convert to DatasetDict and apply subset filtering and limiting
+        test_dataset = DatasetDict.from_dataset(
+            dataset=dataset, subset_list=self.subset_list, limit=self.limit, repeats=self.repeats
+        )
+
+        return test_dataset, None
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        """
+        Convert a raw data record to a Sample object.
+
+        Args:
+            record: Raw data record containing prompt, tags, and metadata
+
+        Returns:
+            Sample: Formatted sample with input messages, theme, and metadata
+        """
+        # Convert prompt messages to chat message objects
+        input_messages = [dict_to_chat_message(message) for message in record['prompt']]
+        # Extract theme from example tags, default to 'Unknown' if no tags
+        tags = record['example_tags']
+        theme = tags[0].split(':')[1].strip() if len(tags) > 0 else 'Unknown'
+        return Sample(input=input_messages, target='', subset_key=theme, metadata=record)
+
+    def llm_match_score(self, original_prediction, filtered_prediction, reference, task_state) -> Score:
+        """
+        Evaluate AI response using LLM judge against physician-created rubrics.
+
+        Args:
+            original_prediction: The AI model's original response
+            filtered_prediction: Filtered/processed version of the response
+            reference: Reference answer (not used in this evaluation)
+            task_state: Contains metadata including rubric items
+
+        Returns:
+            Score: Contains overall score, rubric tag scores, and explanations
+        """
+        from .utils import (
+            RubricItem,
+            calculate_rubric_tag_scores,
+            calculate_score,
+            construct_readable_explanation,
+            parse_json_to_dict,
+        )
+
+        # Initialize the score object with prediction details
+        score = Score(
+            extracted_prediction=filtered_prediction,
+            prediction=original_prediction,
+        )
+
+        # Extract rubric items and conversation from task metadata
+        example = copy.deepcopy(task_state.metadata)
+        rubric_items = [RubricItem.from_dict(d) for d in example['rubrics']]
+        # Construct full conversation including the AI response
+        convo_with_response = example['prompt'] + [dict(content=original_prediction, role='assistant')]
+        # Format conversation as readable string
+        convo_str = '\n\n'.join([f"{m['role']}: {m['content']}" for m in convo_with_response])
+
+        # Evaluate response against each rubric item using LLM judge
+        grading_response_list = []
+        for rubric_item in rubric_items:
+            # Create judge prompt by substituting conversation and rubric item
+            grader_prompt = GRADER_TEMPLATE.replace('<<conversation>>',
+                                                    convo_str).replace('<<rubric_item>>', str(rubric_item))
+            messages = [ChatMessageUser(content=grader_prompt)]
+            # Retry logic for robust evaluation
+            with retry_context(retries=3, sleep_interval=1):
+                grading_response = self.llm_judge.judge(messages=messages)
+                grading_response_dict = parse_json_to_dict(grading_response)
+                # Validate response format and extract boolean criteria_met field
+                if 'criteria_met' in grading_response_dict and isinstance(grading_response_dict['criteria_met'], bool):
+                    grading_response_list.append(grading_response_dict)
+                else:
+                    logger.warning('Grading failed due to bad JSON output, retrying...')
+                    raise ValueError('Grading failed due to bad JSON output')
+
+        # Calculate final scores and explanations
+        overall_score = calculate_score(rubric_items, grading_response_list)  # Overall weighted score
+        rubric_tag_scores, axis_grades = calculate_rubric_tag_scores(
+            rubric_items, grading_response_list
+        )  # Scores by category
+        readable_explanation = construct_readable_explanation(
+            rubric_items, grading_response_list
+        )  # Human-readable results
+
+        # Set score values and metadata
+        score.value = {
+            'overall_score': overall_score,
+            **axis_grades,  # Include axis scores at top level
+        }
+        score.main_score_name = 'overall_score'
+        score.metadata = {
+            'readable_explanation': readable_explanation,
+            'rubric_tag_scores': rubric_tag_scores,
+        }
+        # Store explanation in sample target for reference
+        task_state.target = '**Score Explanation**\n\n' + readable_explanation
+        return score

evalscope/benchmarks/healthbench/utils.py

@@ -0,0 +1,102 @@
+import json
+import re
+from collections import defaultdict
+
+from evalscope.utils import get_logger
+
+logger = get_logger()
+
+
+def parse_json_to_dict(json_string: str) -> dict:
+    # Remove markdown-style ```json``` markers if present
+    json_cleaned = re.sub(r'^```json\s*|\s*```$', '', json_string.strip())
+
+    try:
+        return json.loads(json_cleaned)
+    except json.JSONDecodeError as e:
+        logger.warning(f'JSON decoding failed: {e}')
+        return {}
+
+
+class RubricItem:
+
+    def __init__(self, criterion: str, points: float, tags: list[str]):
+        self.criterion = criterion
+        self.points = points
+        self.tags = tags
+
+    def __str__(self):
+        return f'[{self.points}] {self.criterion}'
+
+    def to_dict(self):
+        return {
+            'criterion': self.criterion,
+            'points': self.points,
+            'tags': self.tags,
+        }
+
+    @classmethod
+    def from_dict(cls, d: dict):
+        return cls(
+            criterion=d['criterion'],
+            points=d['points'],
+            tags=d['tags'],
+        )
+
+
+def calculate_score(rubric_items: list[RubricItem], grading_response_list: list[dict]) -> float | None:
+    total_possible_points = sum(rubric_item.points for rubric_item in rubric_items if rubric_item.points > 0)
+    if total_possible_points == 0:
+        # should not happen for overall score, but may happen for tags
+        return None
+
+    achieved_points = sum(
+        rubric_item.points
+        for rubric_item, grading_response in zip(rubric_items, grading_response_list, strict=True)
+        if grading_response['criteria_met']
+    )
+    overall_score = achieved_points / total_possible_points
+    return overall_score
+
+
+def calculate_rubric_tag_scores(rubric_items: list[RubricItem], grading_response_list: list[dict]) -> dict[str, float]:
+    rubric_tag_items_grades = defaultdict(list)
+    axis_grades = defaultdict(list)
+    for rubric_item, grading_response in zip(rubric_items, grading_response_list):
+        curr_item_tags = set()  # Ensure no duplicates in a rubric item.
+        for tag in rubric_item.tags:
+            rubric_tag_items_grades[tag].append((rubric_item, grading_response))
+            assert tag not in curr_item_tags
+            curr_item_tags.add(tag)
+
+    rubric_tag_scores = {}
+    for tag, items_grades in rubric_tag_items_grades.items():
+        items, grades = zip(*items_grades)
+        score = calculate_score(items, grades)
+        if score is not None:  # implies at least one positive criterion
+            rubric_tag_scores[tag] = score
+            if tag.startswith('axis:'):
+                axis_grades[tag.split(':')[1]] = score
+
+    return rubric_tag_scores, axis_grades
+
+
+def construct_readable_explanation(rubric_items: list[RubricItem], grading_response_list: list[dict]) -> str:
+    rubric_items_with_grades = []
+    readable_explanation_list = []
+    for rubric_item, grading_response in zip(rubric_items, grading_response_list):
+        explanation = grading_response.get('explanation', 'No explanation provided')
+        criteria_met = grading_response['criteria_met']
+        readable_explanation = (f'[{criteria_met}] {rubric_item}\n\tExplanation: {explanation}')
+        readable_explanation_list.append(readable_explanation)
+        rubric_items_with_grades.append({
+            **rubric_item.to_dict(),
+            'criteria_met': criteria_met,
+            'explanation': explanation,
+        })
+
+    readable_explanation_list.sort(key=lambda x: x.startswith('[False]'), reverse=True)
+    readable_explanation_str = '\n\n'.join(readable_explanation_list)
+    readable_explanation_str = f'\n\n{readable_explanation_str}'
+
+    return readable_explanation_str

evalscope/benchmarks/hle/hle_adapter.py

@@ -57,8 +57,9 @@ Your judgment must focus only on if there are meaningful differences between [co
         'humanities/social science (9%), computer science/artificial intelligence (10%), '
         'engineering (4%), chemistry (7%), and other (9%). Around 14% of the questions '
         'require the ability to understand both text and images, i.e., multi-modality. '
-        '24% of the questions are multiple-choice; the rest are short-answer, exact-match questions. '
-        'To evaluate the performance of model without multi-modality capabilities, please set the extra_params["include_multi_modal"] to False.',  # noqa: E501
+        '24% of the questions are multiple-choice; the rest are short-answer, exact-match questions. \n'
+        '**To evaluate the performance of model without multi-modality capabilities, '
+        'please set the `extra_params["include_multi_modal"]` to `False`.**',  # noqa: E501
         dataset_id='cais/hle',
         subset_list=SUBSET_LIST,
         metric_list=['acc'],

evalscope/benchmarks/humaneval/humaneval_adapter.py

@@ -14,9 +14,6 @@ from evalscope.utils.logger import get_logger
 
 logger = get_logger()
 
-# Example:
-# {"task_id": "HumanEval/0", "prompt": "from typing import List\n\n\ndef has_close_elements(numbers: List[float], threshold: float) -> bool:\n    \"\"\" Check if in given list of numbers, are any two numbers closer to each other than\n    given threshold.\n    >>> has_close_elements([1.0, 2.0, 3.0], 0.5)\n    False\n    >>> has_close_elements([1.0, 2.8, 3.0, 4.0, 5.0, 2.0], 0.3)\n    True\n    \"\"\"\n", "entry_point": "has_close_elements", "canonical_solution": "    for idx, elem in enumerate(numbers):\n        for idx2, elem2 in enumerate(numbers):\n            if idx != idx2:\n                distance = abs(elem - elem2)\n                if distance < threshold:\n                    return True\n\n    return False\n", "test": "\n\nMETADATA = {\n    'author': 'jt',\n    'dataset': 'test'\n}\n\n\ndef check(candidate):\n    assert candidate([1.0, 2.0, 3.9, 4.0, 5.0, 2.2], 0.3) == True\n    assert candidate([1.0, 2.0, 3.9, 4.0, 5.0, 2.2], 0.05) == False\n    assert candidate([1.0, 2.0, 5.9, 4.0, 5.0], 0.95) == True\n    assert candidate([1.0, 2.0, 5.9, 4.0, 5.0], 0.8) == False\n    assert candidate([1.0, 2.0, 3.0, 4.0, 5.0, 2.0], 0.1) == True\n    assert candidate([1.1, 2.2, 3.1, 4.1, 5.1], 1.0) == True\n    assert candidate([1.1, 2.2, 3.1, 4.1, 5.1], 0.5) == False\n\n"}  # noqa
-
 
 @register_benchmark(
     BenchmarkMeta(
@@ -24,17 +21,15 @@ logger = get_logger()
         pretty_name='HumanEval',
         tags=[Tags.CODING],
         description=
-        'HumanEval is a benchmark for evaluating the ability of code generation models to write Python functions based on given specifications. It consists of programming tasks with a defined input-output behavior.',
+        'HumanEval is a benchmark for evaluating the ability of code generation models to write Python functions based on given specifications. It consists of programming tasks with a defined input-output behavior. '
+        '**By default the code is executed in local environment. We recommend using sandbox execution to safely run and evaluate the generated code, please refer to the [documentation](https://evalscope.readthedocs.io/en/latest/user_guides/sandbox.html) for more details.**',  # noqa: E501
         dataset_id='opencompass/humaneval',
         subset_list=['openai_humaneval'],
-        metric_list=['Pass@1'],
+        aggregation='mean_and_pass_at_k',
         eval_split='test',
         prompt_template=
         'Read the following function signature and docstring, and fully implement the function described. Your response should only contain the code for this function.\n{question}',
-        extra_params={
-            'num_workers': 4,
-            'timeout': 4
-        },
+        review_timeout=4,
     )
 )
 class HumanevalAdapter(DefaultDataAdapter):
@@ -42,27 +37,6 @@ class HumanevalAdapter(DefaultDataAdapter):
     HumanEval adapter using the new data processing framework.
     """
 
-    def __init__(self, **kwargs):
-        try:
-            from human_eval.data import stream_jsonl, write_jsonl
-            from human_eval.evaluation import check_correctness
-        except ImportError:
-            raise ImportError(
-                'Please install human_eval:'
-                'https://github.com/openai/human-eval/tree/master#installation , '
-                'Note that you need to enable the execution code in the human_eval/execution.py first.'
-            )
-        super().__init__(**kwargs)
-
-        extra_params = kwargs.get('extra_params', {})
-        self.k = [1]
-        self.num_workers = extra_params.get('num_workers', 4)
-        self.timeout = extra_params.get('timeout', 4)
-
-        self.read_problems_func = stream_jsonl
-        self.write_jsonl_func = write_jsonl
-        self.eval_func = check_correctness
-
     def record_to_sample(self, record: Dict[str, Any]) -> Sample:
         """Convert a data record to a Sample object."""
         query = record['prompt']
@@ -94,31 +68,29 @@ class HumanevalAdapter(DefaultDataAdapter):
     def match_score(
         self, original_prediction: str, filtered_prediction: str, reference: str, task_state: TaskState
     ) -> Score:
+
         score = Score(
             extracted_prediction=filtered_prediction,
             prediction=original_prediction,
         )
-
-        # Execute the code and check correctness
-        res = self.eval_func(task_state.metadata, filtered_prediction, self.timeout)
-        passed = res['passed']
-
-        score.value = {'pass': passed}
-        score.explanation = res.get('result', 'Code execution completed')
-        score.metadata = {'task_id': task_state.metadata['task_id'], 'timeout': self.timeout, 'execution_result': res}
-        score.main_score_name = 'pass'
+        problem = task_state.metadata
+        completion = filtered_prediction
+
+        if not self.use_sandbox:
+            from .utils import check_correctness
+
+            # Execute the code and check correctness
+            res = check_correctness(problem=problem, completion=completion, timeout=self.review_timeout)
+            passed = res['passed']
+        else:
+            check_program = (
+                problem['prompt'] + completion + '\n' + problem['test'] + '\n' + f"check({problem['entry_point']})"
+            )
+            res = self.execute_code_in_sandbox(code=check_program, timeout=self.review_timeout, language='python')
+            passed = res.get('status') == 'success'
+        # Set score values
+        score.value = {'acc': passed}
+        score.metadata = {'task_id': problem['task_id'], 'timeout': self.review_timeout, 'execution_result': res}
+        score.main_score_name = 'acc'
 
         return score
-
-    def aggregate_scores(self, sample_scores):
-        from evalscope.metrics.metric import PassAtK
-
-        # caculate pass@k here
-        agg_list = []
-        for metric in self.metric_list:
-            if metric.lower().startswith('pass@'):
-                k = int(metric.split('@')[1])
-                # Get the scores for this metric
-                agg = PassAtK(k)
-                agg_list.extend(agg(sample_scores))
-        return agg_list