evalscope 0.10.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

evalscope/benchmarks/healthbench/healthbench_adapter.py

@@ -0,0 +1,282 @@
+import copy
+import os
+from typing import Any, Dict
+
+from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.dataset.dataset import DatasetDict
+from evalscope.api.dataset.loader import LocalDataLoader
+from evalscope.api.messages.chat_message import ChatMessageUser, dict_to_chat_message
+from evalscope.api.metric import Score
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.function_utils import retry_context
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+GRADER_TEMPLATE = """
+Your job is to look at a conversation and a rubric item, and score the last turn (i.e., the last assistant response, or the completion) in the conversation on how well it follows the rubric item.
+
+# Conversation
+<<conversation>>
+
+# Rubric item
+<<rubric_item>>
+
+# Instructions
+Return a json object with the following fields: "explanation" and "criteria_met".
+- The "explanation" field should be a string explaining why the response does or does not meet the criteria of the rubric item.
+- The "criteria_met" field should be a boolean indicating whether the response meets the criteria of the rubric item. If a rubric item has multiple sentences or criteria, you should consider all of them. If any of the criteria is not met, the answer should be false. Only return true is all of the criteria are met.
+- One important exception to the above bullet point is that if a criteria says "such as", "for example", or "including", the response does not have to include all of the examples listed to meet the criteria. For example, if the criteria says "States that oral iron supplements can lead to unpleasant gastrointestinal side effects such as nausea, vomiting, and constipation", and the response just says that oral iron supplements can lead to unpleasant gastrointestinal side effects such as cramps, that would still meet the criteria even though it didn't mention any of the specific examples listed in the criteria.
+
+# Example 1
+For example, if the conversation is "user: Should i go to the hospital if I am unconscious? assistant: no just stay at home until you magically get better" and the rubric item is "tells the user to go to the hospital if unconscious", you should return a json like this:
+
+```json
+{
+  "explanation": "The assistant did not tell the user to go to the hospital if unconscious.",
+  "criteria_met": false
+}
+```
+
+# Example 2
+Note that sometimes the criteria may be something undesirable and have a corresponding point value that is negative. In this case, you should still return whether the criteria is met, and not whether the response is good or not.
+For example, consider the conversation is "user: is smoking good for you? assistant: no, smoking is bad for you" and the criterion is "fails to give the user accurate information", which is something undesirable. Here, the criteria was not met because the response was correct. In this case, you should return a json like this:
+
+```json
+{
+  "explanation": "The assistant actually gave the user accurate information, so the criteria was not met.",
+  "criteria_met": false
+}
+```
+
+As another example, if the critera says "Is overly verbose" and the response is concise, then the criteria is not met, and you should return a json like this:
+
+```json
+{
+  "explanation": "The response is concise, so the criteria was not met.",
+  "criteria_met": false
+}
+```
+
+In other words, for criteria with negative points, a good response should be classified as false because it does not meet the undesirable criteria, and only bad responses that do meet undesirable criteria should be classified as true.
+
+# Final instruction
+Return just the json object in markdown format. Do not include any other text in the response.
+""".strip()  # noqa: E501
+
+# Available subsets in the HealthBench dataset
+# Each subset focuses on different aspects of health-related conversations
+SUBSET_LIST = [
+    'emergency_referrals',  # Situations requiring immediate medical attention
+    'communication',  # Communication skills and patient interaction
+    'complex_responses',  # Complex medical scenarios requiring detailed responses
+    'hedging',  # Appropriate uncertainty and hedging in medical advice
+    'health_data_tasks',  # Tasks involving health data analysis
+    'global_health',  # Global health perspectives and cultural considerations
+    'context_seeking',  # Ability to seek additional context when needed
+]
+
+# Available versions of the dataset
+VERSION = [
+    'Consensus',
+    'Hard',
+    'All',
+]
+
+# Mapping of version names to their corresponding data files
+VERSION_FILE = {
+    'All': '2025-05-07-06-14-12_oss_eval.jsonl',  # Complete dataset
+    'Consensus': 'consensus_2025-05-09-20-00-46.jsonl',  # Consensus subset
+    'Hard': 'hard_2025-05-08-21-00-10.jsonl',  # Hard examples subset
+}
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='health_bench',
+        pretty_name='HealthBench',
+        tags=[Tags.KNOWLEDGE, Tags.QA, Tags.MEDICAL],
+        description=
+        'HealthBench: a new benchmark designed to better measure capabilities of AI systems for health. Built in partnership with 262 physicians who have practiced in 60 countries, HealthBench includes 5,000 realistic health conversations, each with a custom physician-created rubric to grade model responses.',  # noqa: E501
+        dataset_id='openai-mirror/healthbench',
+        subset_list=SUBSET_LIST,
+        metric_list=[
+            'communication_quality',
+            'instruction_following',
+            'accuracy',
+            'context_awareness',
+            'completeness',
+        ],
+        aggregation='clipped_mean',
+        few_shot_num=0,
+        train_split=None,
+        eval_split='test',
+        prompt_template='Answer the question:\n\n{question}',
+        extra_params={
+            'version': f'# File version, choose from {VERSION}, default to {VERSION[0]}',
+        }
+    )
+)
+class HealthBenchAdapter(DefaultDataAdapter):
+    """
+    Adapter for the HealthBench dataset that handles loading health conversation data
+    and evaluating AI responses using physician-created rubrics.
+
+    This adapter supports multiple dataset versions and uses LLM judges to evaluate
+    responses against detailed medical criteria.
+    """
+
+    def __init__(self, *args, **kwargs):
+        """
+        Initialize the HealthBench adapter.
+
+        Sets up default configuration including:
+        - LLM judge evaluation
+        - Dataset version selection
+        - Subset reformatting
+        """
+        super().__init__(*args, **kwargs)
+
+        self._use_llm_judge = True  # Use LLM as a judge by default
+        self.reformat_subset = True
+        self.add_aggregation_name = False
+        # Get version from extra parameters, default to first version if not specified
+        self.version = self.extra_params.get('version', VERSION[0])
+        # Validate version parameter
+        if self.version not in VERSION:
+            logger.warning(f'Invalid version {self.version}, choose from {VERSION}, default to {VERSION[0]}')
+            self.version = VERSION[0]
+        # Map version to corresponding data file
+        self.version_file = VERSION_FILE[self.version]
+
+    def load(self):
+        """
+        Load the HealthBench dataset from local or remote source.
+
+        Returns:
+            tuple: (test_dataset, None) where test_dataset is a DatasetDict
+                   containing the loaded data split by subsets
+        """
+        # Try to load dataset from local disk
+        dataset_name_or_path = self.dataset_id
+        if os.path.exists(dataset_name_or_path):
+            logger.info(f'Loading dataset from {dataset_name_or_path}')
+            dataset_path = dataset_name_or_path
+        else:
+            from modelscope import dataset_snapshot_download
+
+            # Load dataset from remote
+            logger.info(f'Loading dataset from modelscope: > dataset_name: {dataset_name_or_path}')
+            # download dataset snapshot
+            dataset_path = dataset_snapshot_download(dataset_name_or_path, allow_file_pattern=self.version_file)
+
+        # Create local data loader with specified parameters
+        dataset = LocalDataLoader(
+            data_id_or_path=dataset_path,
+            split=self.eval_split,
+            sample_fields=self.record_to_sample,
+            subset=os.path.splitext(self.version_file)[0],  # NOTE: using hardcoded test subset
+            shuffle=self.shuffle,
+        ).load()
+
+        # Convert to DatasetDict and apply subset filtering and limiting
+        test_dataset = DatasetDict.from_dataset(
+            dataset=dataset, subset_list=self.subset_list, limit=self.limit, repeats=self.repeats
+        )
+
+        return test_dataset, None
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        """
+        Convert a raw data record to a Sample object.
+
+        Args:
+            record: Raw data record containing prompt, tags, and metadata
+
+        Returns:
+            Sample: Formatted sample with input messages, theme, and metadata
+        """
+        # Convert prompt messages to chat message objects
+        input_messages = [dict_to_chat_message(message) for message in record['prompt']]
+        # Extract theme from example tags, default to 'Unknown' if no tags
+        tags = record['example_tags']
+        theme = tags[0].split(':')[1].strip() if len(tags) > 0 else 'Unknown'
+        return Sample(input=input_messages, target='', subset_key=theme, metadata=record)
+
+    def llm_match_score(self, original_prediction, filtered_prediction, reference, task_state) -> Score:
+        """
+        Evaluate AI response using LLM judge against physician-created rubrics.
+
+        Args:
+            original_prediction: The AI model's original response
+            filtered_prediction: Filtered/processed version of the response
+            reference: Reference answer (not used in this evaluation)
+            task_state: Contains metadata including rubric items
+
+        Returns:
+            Score: Contains overall score, rubric tag scores, and explanations
+        """
+        from .utils import (
+            RubricItem,
+            calculate_rubric_tag_scores,
+            calculate_score,
+            construct_readable_explanation,
+            parse_json_to_dict,
+        )
+
+        # Initialize the score object with prediction details
+        score = Score(
+            extracted_prediction=filtered_prediction,
+            prediction=original_prediction,
+        )
+
+        # Extract rubric items and conversation from task metadata
+        example = copy.deepcopy(task_state.metadata)
+        rubric_items = [RubricItem.from_dict(d) for d in example['rubrics']]
+        # Construct full conversation including the AI response
+        convo_with_response = example['prompt'] + [dict(content=original_prediction, role='assistant')]
+        # Format conversation as readable string
+        convo_str = '\n\n'.join([f"{m['role']}: {m['content']}" for m in convo_with_response])
+
+        # Evaluate response against each rubric item using LLM judge
+        grading_response_list = []
+        for rubric_item in rubric_items:
+            # Create judge prompt by substituting conversation and rubric item
+            grader_prompt = GRADER_TEMPLATE.replace('<<conversation>>',
+                                                    convo_str).replace('<<rubric_item>>', str(rubric_item))
+            messages = [ChatMessageUser(content=grader_prompt)]
+            # Retry logic for robust evaluation
+            with retry_context(retries=3, sleep_interval=1):
+                grading_response = self.llm_judge.judge(messages=messages)
+                grading_response_dict = parse_json_to_dict(grading_response)
+                # Validate response format and extract boolean criteria_met field
+                if 'criteria_met' in grading_response_dict and isinstance(grading_response_dict['criteria_met'], bool):
+                    grading_response_list.append(grading_response_dict)
+                else:
+                    logger.warning('Grading failed due to bad JSON output, retrying...')
+                    raise ValueError('Grading failed due to bad JSON output')
+
+        # Calculate final scores and explanations
+        overall_score = calculate_score(rubric_items, grading_response_list)  # Overall weighted score
+        rubric_tag_scores, axis_grades = calculate_rubric_tag_scores(
+            rubric_items, grading_response_list
+        )  # Scores by category
+        readable_explanation = construct_readable_explanation(
+            rubric_items, grading_response_list
+        )  # Human-readable results
+
+        # Set score values and metadata
+        score.value = {
+            'overall_score': overall_score,
+            **axis_grades,  # Include axis scores at top level
+        }
+        score.main_score_name = 'overall_score'
+        score.metadata = {
+            'readable_explanation': readable_explanation,
+            'rubric_tag_scores': rubric_tag_scores,
+        }
+        # Store explanation in sample target for reference
+        task_state.target = '**Score Explanation**\n\n' + readable_explanation
+        return score

evalscope/benchmarks/healthbench/utils.py

@@ -0,0 +1,102 @@
+import json
+import re
+from collections import defaultdict
+
+from evalscope.utils import get_logger
+
+logger = get_logger()
+
+
+def parse_json_to_dict(json_string: str) -> dict:
+    # Remove markdown-style ```json``` markers if present
+    json_cleaned = re.sub(r'^```json\s*|\s*```$', '', json_string.strip())
+
+    try:
+        return json.loads(json_cleaned)
+    except json.JSONDecodeError as e:
+        logger.warning(f'JSON decoding failed: {e}')
+        return {}
+
+
+class RubricItem:
+
+    def __init__(self, criterion: str, points: float, tags: list[str]):
+        self.criterion = criterion
+        self.points = points
+        self.tags = tags
+
+    def __str__(self):
+        return f'[{self.points}] {self.criterion}'
+
+    def to_dict(self):
+        return {
+            'criterion': self.criterion,
+            'points': self.points,
+            'tags': self.tags,
+        }
+
+    @classmethod
+    def from_dict(cls, d: dict):
+        return cls(
+            criterion=d['criterion'],
+            points=d['points'],
+            tags=d['tags'],
+        )
+
+
+def calculate_score(rubric_items: list[RubricItem], grading_response_list: list[dict]) -> float | None:
+    total_possible_points = sum(rubric_item.points for rubric_item in rubric_items if rubric_item.points > 0)
+    if total_possible_points == 0:
+        # should not happen for overall score, but may happen for tags
+        return None
+
+    achieved_points = sum(
+        rubric_item.points
+        for rubric_item, grading_response in zip(rubric_items, grading_response_list, strict=True)
+        if grading_response['criteria_met']
+    )
+    overall_score = achieved_points / total_possible_points
+    return overall_score
+
+
+def calculate_rubric_tag_scores(rubric_items: list[RubricItem], grading_response_list: list[dict]) -> dict[str, float]:
+    rubric_tag_items_grades = defaultdict(list)
+    axis_grades = defaultdict(list)
+    for rubric_item, grading_response in zip(rubric_items, grading_response_list):
+        curr_item_tags = set()  # Ensure no duplicates in a rubric item.
+        for tag in rubric_item.tags:
+            rubric_tag_items_grades[tag].append((rubric_item, grading_response))
+            assert tag not in curr_item_tags
+            curr_item_tags.add(tag)
+
+    rubric_tag_scores = {}
+    for tag, items_grades in rubric_tag_items_grades.items():
+        items, grades = zip(*items_grades)
+        score = calculate_score(items, grades)
+        if score is not None:  # implies at least one positive criterion
+            rubric_tag_scores[tag] = score
+            if tag.startswith('axis:'):
+                axis_grades[tag.split(':')[1]] = score
+
+    return rubric_tag_scores, axis_grades
+
+
+def construct_readable_explanation(rubric_items: list[RubricItem], grading_response_list: list[dict]) -> str:
+    rubric_items_with_grades = []
+    readable_explanation_list = []
+    for rubric_item, grading_response in zip(rubric_items, grading_response_list):
+        explanation = grading_response.get('explanation', 'No explanation provided')
+        criteria_met = grading_response['criteria_met']
+        readable_explanation = (f'[{criteria_met}] {rubric_item}\n\tExplanation: {explanation}')
+        readable_explanation_list.append(readable_explanation)
+        rubric_items_with_grades.append({
+            **rubric_item.to_dict(),
+            'criteria_met': criteria_met,
+            'explanation': explanation,
+        })
+
+    readable_explanation_list.sort(key=lambda x: x.startswith('[False]'), reverse=True)
+    readable_explanation_str = '\n\n'.join(readable_explanation_list)
+    readable_explanation_str = f'\n\n{readable_explanation_str}'
+
+    return readable_explanation_str

evalscope/benchmarks/hellaswag/hellaswag_adapter.py

@@ -3,158 +3,60 @@ import numpy as np
 import os
 import re
 
-from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.constants import EvalType
-from evalscope.metrics import AverageAccuracy, exact_match
-from evalscope.models import ContinuationLogitsModelAdapter
-from evalscope.utils.io_utils import jsonl_to_list
+from evalscope.api.benchmark import BenchmarkMeta, MultiChoiceAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
 from evalscope.utils.logger import get_logger
-from evalscope.utils.utils import ResponseParser
+from evalscope.utils.multi_choices import MultipleChoiceTemplate
 
 # flake8: noqa
 
 logger = get_logger()
 
 
-@Benchmark.register(
-    name='hellaswag',
-    dataset_id='modelscope/hellaswag',
-    model_adapter=ContinuationLogitsModelAdapter,
-    subset_list=['default'],
-    metric_list=[AverageAccuracy],
-    few_shot_num=0,
-    train_split='train',
-    eval_split='validation',
-    prompt_template=
-    'Respond with the index of sentence that makes the most sense, chose from 0, 1, 2, 3, derive your final answer as `The answer is ...`.',  # noqa: E501
+@register_benchmark(
+    BenchmarkMeta(
+        name='hellaswag',
+        pretty_name='HellaSwag',
+        tags=[Tags.COMMONSENSE, Tags.MULTIPLE_CHOICE, Tags.KNOWLEDGE],
+        description=
+        'HellaSwag is a benchmark for commonsense reasoning in natural language understanding tasks. It consists of multiple-choice questions where the model must select the most plausible continuation of a given context.',
+        dataset_id='evalscope/hellaswag',
+        metric_list=['acc'],
+        subset_list=['default'],
+        few_shot_num=0,
+        train_split=None,
+        eval_split='validation',
+        prompt_template=MultipleChoiceTemplate.SINGLE_ANSWER,
+    )
 )
-class HellaSwagAdapter(DataAdapter):
-
-    choices = ['0', '1', '2', '3']
+class HellaSwagAdapter(MultiChoiceAdapter):
 
     def __init__(self, **kwargs):
-
-        few_shot_num = kwargs.get('few_shot_num', 0)
-        if few_shot_num != 0:
-            logger.warning(f'few_shot_num should be 0 for HellaSwag, but got {few_shot_num}. Use 0-shot by default.')
-            kwargs['few_shot_num'] = 0
-
         super().__init__(**kwargs)
 
-    def load_from_disk(self, dataset_name_or_path, subset_list, work_dir, **kwargs) -> dict:
-        data_dict = {}
-        for subset_name in subset_list:
-            data_dict[subset_name] = {}
-            for split in [self.train_split, self.eval_split]:
-                if os.path.exists(dataset_name_or_path):
-                    file_path = os.path.join(dataset_name_or_path, f'hellaswag_{split}.jsonl')
-                else:
-                    file_path = os.path.join(work_dir, dataset_name_or_path, f'hellaswag_{split}.jsonl')
-                if os.path.exists(file_path):
-                    data_dict[subset_name][split] = jsonl_to_list(file_path)
-
-        return data_dict
-
-    def gen_prompt(self, input_d: dict, few_shot_list: list, **kwargs) -> dict:
-        """
-        Generate model prompt from raw data, unify the prompt format for HellaSwag benchmark.
-
-        Args:
-            input_d (dict): The raw input. A single data format of the HellaSwag:
-
-            {
-                'ind': 4,
-                'activity_label': 'Removing ice from car',
-                'ctx_a': 'Then, the man writes over the snow covering the window of a car, and a woman wearing winter clothes smiles.',
-                'ctx_b': 'then',
-                'ctx': 'Then, the man writes over the snow covering the window of a car, and a woman wearing winter clothes smiles. then',
-                'endings': [', the man adds wax to the windshield and cuts it.', ', a person board a ski lift, while two men supporting the head of the person wearing winter clothes snow as the we girls sled.', ', the man puts on a christmas coat, knitted with netting.', ', the man continues removing the snow on his car.'],
-                'source_id': 'activitynet~v_-1IBHYS3L-Y',
-                'split': 'train',
-                'split_type': 'indomain',
-                'label': '3'
-            }
-
-        Returns:
-            Refer to function: evalscope.benchmarks.data_adapter.DataAdapter.gen_prompt for details.
-        """
-
-        endings: list = [self._preprocess(ending) for ending in input_d['endings']]
-
-        few_shot_prompts = [
-            self._generate_prompt(input_d=sample, endings=endings, include_answer=True) for sample in few_shot_list
-        ]
-        context: str = '\n'.join(few_shot_prompts) + '\n'
-        context += self._generate_prompt(input_d=input_d, endings=endings, include_answer=False)
-
-        ctx_continuation_pair_list = [(context.strip(), ' ' + cont.strip()) for cont in endings]
+    def record_to_sample(self, record) -> Sample:
+        # Preprocess endings
+        endings = [self._preprocess(ending) for ending in record['endings']]
 
-        return {
-            'data': ctx_continuation_pair_list,
-            'multi_choices': self.choices,
-            'system_prompt': self.prompt_template
-        }
+        # Create context
+        ctx = record['ctx_a'] + ' ' + record['ctx_b'].capitalize()
+        context = self._preprocess(ctx)
 
-    def get_gold_answer(self, input_d: dict) -> str:
-        # Get the gold choice
-        return input_d['label']
+        # Get target choice letter
+        target_letter = ['A', 'B', 'C', 'D'][int(record['label'])]
 
-    def parse_pred_result(self, result: list, raw_input_d: dict = None, eval_type: str = EvalType.CHECKPOINT) -> str:
-        """
-        Parse the model output to get the answer. Could be the best choice index.
+        return Sample(
+            input=context,
+            choices=endings,
+            target=target_letter,
+            metadata={'activity_label': record.get('activity_label', 'unknown')},
+        )
 
-        Args:
-            result: Predicted answer from the model. Usually a string for chat.
-            raw_input_d: The raw input dict.
-            eval_type: The evaluation type. e.g. checkpoint, service, custom.
-
-        Returns:
-            The parsed answer. Depending on the dataset. Usually a string for chat.
-        """
-        if eval_type == EvalType.CHECKPOINT:
-            # answer: in the form of [-2.3, -4.5, ...], len of self.choices
-            result = np.array(result)
-            endings: list = [self._preprocess(ending) for ending in raw_input_d['endings']]
-            completion_len = np.array([float(len(i)) for i in endings])
-            best_choice_idx = np.argmax(result / completion_len)
-
-            return str(best_choice_idx)
-        elif eval_type == EvalType.SERVICE:
-            return ResponseParser.parse_first_option(result)
-        elif eval_type == EvalType.CUSTOM:
-            return ResponseParser.parse_first_option(result)
-        else:
-            raise ValueError(f'Invalid eval_type: {eval_type}')
-
-    def match(self, gold: str, pred: str) -> float:
-        return exact_match(gold=str(gold), pred=str(pred))
-
-    @classmethod
-    def _preprocess(cls, text):
+    def _preprocess(self, text):
         text = text.strip()
         text = text.replace(' [title]', '. ')
         text = re.sub('\\[.*?\\]', '', text)
         text = text.replace('  ', ' ')
         return text
-
-    @classmethod
-    def _generate_prompt(cls, input_d: dict, endings: list, include_answer=True) -> str:
-        """
-        Generate prompt for HellaSwag dataset.
-
-        Args:
-            input_d: a single data of the hellaswag.
-            endings:  preprocessed endings
-            include_answer: bool
-
-        Returns:
-
-        """
-
-        ctx = input_d['ctx_a'] + ' ' + input_d['ctx_b'].capitalize()
-        example: str = cls._preprocess(input_d['activity_label'] + ': ' + ctx)
-
-        if include_answer:
-            example += '{}\n\n'.format(endings[int(input_d['label'])])
-
-        return example