evalscope 1.0.0__py3-none-any.whl → 1.0.2__py3-none-any.whl

evalscope/benchmarks/bfcl/bfcl_adapter.py

@@ -1,17 +1,19 @@
-import importlib
 import json
 import re
 import traceback
-from typing import Any, Dict
+from typing import Any, Dict, List
 
 from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter
 from evalscope.api.dataset import Sample
 from evalscope.api.evaluator import TaskState
 from evalscope.api.messages.chat_message import ChatMessageUser
 from evalscope.api.metric import Score
+from evalscope.api.metric.scorer import AggScore
 from evalscope.api.model import Model, ModelOutput
 from evalscope.api.registry import register_benchmark
 from evalscope.constants import Tags
+from evalscope.report import Category, Report, Subset
+from evalscope.utils.import_utils import check_import
 from evalscope.utils.logger import get_logger
 
 logger = get_logger()
@@ -67,18 +69,50 @@ class BFCLAdapter(DefaultDataAdapter):
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
 
-        spec = importlib.util.find_spec('bfcl_eval')
-        if spec is None:
-            raise ImportError(
-                '`bfcl_eval` not found, please install it with `pip install bfcl-eval==2025.6.16` before evaluating.'
-            )
+        check_import('bfcl_eval', package='bfcl-eval==2025.6.16', raise_error=True, feature_name=self.pretty_name)
 
         self.category_map = SUBJECT_MAPPING
         self.reformat_subset = True
+        self.add_overall_metric = False
+        self.add_aggregation_name = False
 
         self.underscore_to_dot = self.extra_params.get('underscore_to_dot', True)
         self.is_fc_model = self.extra_params.get('is_fc_model', True)
 
+    def _weighted_average_from_subsets(self, subset_names: List[str], subset_dict: Dict[str, Subset]) -> Subset:
+        """Calculate weighted average for given subsets.
+
+        Returns:
+            Subset: A new Subset object with weighted average score
+        """
+        total_score = 0
+        total_count = 0
+        for name in subset_names:
+            if name in subset_dict:
+                subset = subset_dict[name]
+                total_score += subset.score * subset.num
+                total_count += subset.num
+
+        weighted_avg = total_score / total_count if total_count > 0 else 0
+        return Subset(name='', score=weighted_avg, num=total_count)
+
+    def _unweighted_average_from_subsets(self, subset_names: List[str], subset_dict: Dict[str, Subset]) -> Subset:
+        """Calculate unweighted average for given subsets.
+
+        Returns:
+            Subset: A new Subset object with unweighted average score
+        """
+        scores = []
+        total_count = 0
+        for name in subset_names:
+            if name in subset_dict:
+                subset = subset_dict[name]
+                scores.append(subset.score)
+                total_count += subset.num
+
+        unweighted_avg = sum(scores) / len(scores) if scores else 0
+        return Subset(name='', score=unweighted_avg, num=total_count)
+
     def preprocess_row(self, row: dict):
         """
         Inplace preprocess the row to ensure it has the correct format for BFCL evaluation.
@@ -256,3 +290,104 @@ class BFCLAdapter(DefaultDataAdapter):
             score.metadata = {'error': traceback.format_exc()}
             score.main_score_name = 'acc'
         return score
+
+    def _on_generate_report_end(self, report: Report, output_dir, **kwargs):
+        """
+        Finalize the report generation process. Calculate the overall score.
+
+        Track the number of each category.
+        - step1: simple, java, javascript unweighted average as simple_ast
+        - step2.1: simple_ast, multiple, parallel, parallel_multiple unweighted average as ast_non_live
+        - step2.2: live_simple, live_multiple, live_parallel, live_parallel_multiple weighted average as ast_live
+        - step2.3: irrelevance as hallucination_non_live
+        - step2.4: live_irrelevance, live_relevance weighted average as hallucination_live
+        - step2.5: multi_turn_base as multi_turn_base
+        - step2.6: multi_turn_miss_func, multi_turn_miss_param, multi_turn_long_context weighted average as multi_turn_augmented
+        - step3.1: ast_non_live, hallucination_non_live unweighted average as non_live
+        - step3.2: ast_live, hallucination_live weighted average as live
+        - step3.3: multi_turn_base, multi_turn_augmented unweighted average as multi_turn
+        - step4: non_live, live, multi_turn unweighted average as overall
+        Args:
+            report (Report): The generated evaluation report.
+            output_dir (str): The directory to save the report.
+
+        Returns:
+            None
+        """  # noqa: E501
+        for metric in report.metrics:
+            # Collect all subsets in a dictionary for easy access
+            subset_dict: Dict[str, Subset] = {}
+            for category in metric.categories:
+                for subset in category.subsets:
+                    subset_dict[subset.name] = subset
+
+            # Step 1: Calculate simple_ast (simple, java, javascript unweighted average)
+            simple_subsets = ['simple', 'java', 'javascript']
+            simple_ast = self._unweighted_average_from_subsets(simple_subsets, subset_dict)
+            subset_dict['simple_ast'] = simple_ast
+
+            # Step 2.1: Calculate ast_non_live
+            # (simple_ast, multiple, parallel, parallel_multiple unweighted average)
+            ast_non_live_subsets = ['simple_ast', 'multiple', 'parallel', 'parallel_multiple']
+            ast_non_live = self._unweighted_average_from_subsets(ast_non_live_subsets, subset_dict)
+            subset_dict['ast_non_live'] = ast_non_live
+
+            # Step 2.2: Calculate ast_live
+            # (live_simple, live_multiple, live_parallel, live_parallel_multiple weighted average)
+            live_subsets = ['live_simple', 'live_multiple', 'live_parallel', 'live_parallel_multiple']
+            ast_live = self._weighted_average_from_subsets(live_subsets, subset_dict)
+            subset_dict['ast_live'] = ast_live
+
+            # Step 2.3: hallucination_non_live (irrelevance)
+            if 'irrelevance' in subset_dict:
+                subset_dict['hallucination_non_live'] = subset_dict['irrelevance']
+            else:
+                subset_dict['hallucination_non_live'] = Subset(name='hallucination_non_live', score=0, num=0)
+
+            # Step 2.4: Calculate hallucination_live (live_irrelevance, live_relevance weighted average)
+            hallucination_live_subsets = ['live_irrelevance', 'live_relevance']
+            hallucination_live = self._weighted_average_from_subsets(hallucination_live_subsets, subset_dict)
+            subset_dict['hallucination_live'] = hallucination_live
+
+            # Step 2.5: multi_turn_base
+            if 'multi_turn_base' not in subset_dict:
+                subset_dict['multi_turn_base'] = Subset(name='multi_turn_base', score=0, num=0)
+
+            # Step 2.6: Calculate multi_turn_augmented
+            # (multi_turn_miss_func, multi_turn_miss_param, multi_turn_long_context weighted average)
+            multi_turn_augmented_subsets = ['multi_turn_miss_func', 'multi_turn_miss_param', 'multi_turn_long_context']
+            multi_turn_augmented = self._weighted_average_from_subsets(multi_turn_augmented_subsets, subset_dict)
+            subset_dict['multi_turn_augmented'] = multi_turn_augmented
+
+            # Step 3.1: Calculate non_live (ast_non_live, hallucination_non_live unweighted average)
+            non_live_subsets = ['ast_non_live', 'hallucination_non_live']
+            non_live = self._unweighted_average_from_subsets(non_live_subsets, subset_dict)
+            subset_dict['non_live'] = non_live
+
+            # Step 3.2: Calculate live (ast_live, hallucination_live weighted average)
+            live_agg_subsets = ['ast_live', 'hallucination_live']
+            live = self._weighted_average_from_subsets(live_agg_subsets, subset_dict)
+            subset_dict['live'] = live
+
+            # Step 3.3: Calculate multi_turn (multi_turn_base, multi_turn_augmented unweighted average)
+            multi_turn_subsets = ['multi_turn_base', 'multi_turn_augmented']
+            multi_turn = self._unweighted_average_from_subsets(multi_turn_subsets, subset_dict)
+            subset_dict['multi_turn'] = multi_turn
+
+            # Step 4: Calculate overall (non_live, live, multi_turn unweighted average)
+            overall_subsets = ['non_live', 'live', 'multi_turn']
+            overall = self._unweighted_average_from_subsets(overall_subsets, subset_dict)
+            subset_dict['overall'] = overall
+
+            # Add computed scores to the category
+            computed_subset_names = ['non_live', 'live', 'multi_turn', 'overall']
+
+            # Add the computed scores as new subsets in the metric
+            dummy_subsets = []
+            for subset_name in computed_subset_names:
+                if subset_name in subset_dict:
+                    subset = subset_dict[subset_name]
+                    subset.name = subset_name.upper()
+                    dummy_subsets.append(subset)
+            dummy_category = Category(name='-', subsets=dummy_subsets)
+            metric.categories.append(dummy_category)

evalscope/benchmarks/bfcl/generation.py

@@ -72,13 +72,14 @@ def generate_turn(model: Model, row: dict[str, Any]):
 
             # Handle the response based on the model output structure
             message = model_output.message
-            model_usage += model_output.usage
+            if model_output.usage is not None:
+                model_usage += model_output.usage
 
             current_messages.append(message)
             if isinstance(message, str):
                 result = message
             else:
-                result = message.content
+                result = message.text
 
             logger.debug(f'Turn:{turn_idx} Step:{n_steps} Result: {result}')
             current_responses.append(result)
@@ -115,7 +116,7 @@ def generate_turn(model: Model, row: dict[str, Any]):
 
             n_steps += 1
             if n_steps > MAXIMUM_STEP_LIMIT:
-                logger.error(f'INFERENCE_ERROR: Exceeded max inference steps ({MAXIMUM_STEP_LIMIT})')
+                logger.warning(f'INFERENCE_WARNING: Exceeded max inference steps ({MAXIMUM_STEP_LIMIT})')
                 break
 
         all_model_responses.append(current_responses)
@@ -145,9 +146,7 @@ def generate_turn_with_tools(model: Model, row: dict[str, Any]):
             new_tools = row['missing_functions'][str(turn_idx)]
             for new_tool in new_tools:
                 cur_tool = new_tool[0]
-                # change type to object
-                if cur_tool['parameters']['type'] != 'object':
-                    cur_tool['parameters']['type'] = 'object'
+                cur_tool['parameters']['type'] = 'object'
                 tools.append({
                     'type': 'function',
                     'function': cur_tool,
@@ -172,7 +171,8 @@ def generate_turn_with_tools(model: Model, row: dict[str, Any]):
 
             # Handle the response based on the model output structure
             message = model_output.message
-            model_usage += model_output.usage
+            if model_output.usage is not None:
+                model_usage += model_output.usage
 
             current_messages.append(message)
             if isinstance(message, str):
@@ -186,7 +186,7 @@ def generate_turn_with_tools(model: Model, row: dict[str, Any]):
                     logger.error(f'Error converting tool calls to function call strings: {e}')
                     tool_call_strs = None
             else:
-                model_responses = [message.content]
+                model_responses = [message.text]
                 tool_call_strs = None
 
             current_responses.extend(model_responses)
@@ -214,7 +214,7 @@ def generate_turn_with_tools(model: Model, row: dict[str, Any]):
 
             n_steps += 1
             if n_steps > MAXIMUM_STEP_LIMIT:
-                logger.error(f'INFERENCE_ERROR: Exceeded max inference steps ({MAXIMUM_STEP_LIMIT})')
+                logger.warning(f'INFERENCE_WARNING: Exceeded max inference steps ({MAXIMUM_STEP_LIMIT})')
                 break
 
         all_model_responses.append(current_responses)

evalscope/benchmarks/ceval/ceval_adapter.py

@@ -1,10 +1,9 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 
-from functools import partial
 from typing import Any, Dict
 
 from evalscope.api.benchmark import BenchmarkMeta, MultiChoiceAdapter
-from evalscope.api.dataset import Dataset, RemoteDataLoader, Sample
+from evalscope.api.dataset import Sample
 from evalscope.api.registry import register_benchmark
 from evalscope.constants import Tags
 from evalscope.utils.logger import get_logger

evalscope/benchmarks/data_collection/data_collection_adapter.py

@@ -6,9 +6,7 @@ from typing import Any, Dict, List
 from evalscope.api.benchmark import BenchmarkMeta, DataAdapter, DefaultDataAdapter
 from evalscope.api.dataset import DatasetDict, LocalDataLoader, Sample
 from evalscope.api.evaluator import TaskState
-from evalscope.api.metric import Score
 from evalscope.api.metric.scorer import AggScore, SampleScore
-from evalscope.api.model.model import Model
 from evalscope.api.registry import get_benchmark, register_benchmark
 from evalscope.config import TaskConfig
 from evalscope.constants import DataCollection, Tags
@@ -23,7 +21,11 @@ logger = get_logger()
     BenchmarkMeta(
         name=DataCollection.NAME,
         dataset_id='',  # dataset_id need to be set
-        description='Data collection',
+        description='Custom Data collection, mixing multiple evaluation datasets for '
+        'a unified evaluation, aiming to use less data to achieve a more comprehensive '
+        'assessment of the model\'s capabilities. '
+        '[Usage Reference](https://evalscope.readthedocs.io/zh-cn/latest/advanced_guides/collection/index.html)',
+        tags=[Tags.CUSTOM],
         metric_list=['acc'],
         eval_split='test',
         prompt_template='',
@@ -55,9 +57,10 @@ class DataCollectionAdapter(DefaultDataAdapter):
             data_id_or_path=dataset_path,
             split=self.eval_split,
             sample_fields=self.record_to_sample,
-            subset=self.default_subset,
+            subset='test',  # NOTE: using hardcoded test subset
             limit=self.limit,
-            repeats=self.repeats
+            repeats=self.repeats,
+            shuffle=self.shuffle,
         ).load()
 
         test_dataset = DatasetDict({self.default_subset: dataset})
@@ -95,7 +98,6 @@ class DataCollectionAdapter(DefaultDataAdapter):
 
         # load dataset args
         dataset_args = copy.deepcopy(self._task_config.dataset_args)
-        common_args = dataset_args.get(DataCollection.NAME, {})
 
         # Iterate through each sample in the dataset
         dataset = self.test_dataset[self.default_subset]
@@ -108,7 +110,6 @@ class DataCollectionAdapter(DefaultDataAdapter):
 
             # update dataset args
             cur_dataset_args = dataset_args.get(dataset_name, {})
-            cur_dataset_args.update(common_args)
 
             # Initialize dataset adapter
             if dataset_name not in self.dataset_adapters:
@@ -141,19 +142,22 @@ class DataCollectionAdapter(DefaultDataAdapter):
         data = []
         for sample_score in sample_scores:
             collection_info = sample_score.sample_metadata[DataCollection.INFO]
-            for metric_name, value in sample_score.score.value.items():
-                data.append(
-                    dict(
-                        task_type=collection_info['task_type'],
-                        categories=tuple(collection_info['categories']),
-                        dataset_name=collection_info['dataset_name'],
-                        subset_name=collection_info['subset_name'],
-                        tags=collection_info['tags'],
-                        sample_id=sample_score.sample_id,
-                        metric=metric_name,
-                        score=value
-                    )
+            main_score = sample_score.score.main_value
+            main_metric = sample_score.score.main_score_name
+
+            # use main score
+            data.append(
+                dict(
+                    task_type=collection_info['task_type'],
+                    categories=tuple(collection_info['categories']),
+                    dataset_name=collection_info['dataset_name'],
+                    subset_name=collection_info['subset_name'],
+                    tags=collection_info['tags'],
+                    sample_id=sample_score.sample_id,
+                    metric=main_metric,
+                    score=main_score
                 )
+            )
 
         df = pd.DataFrame(data)
 

evalscope/benchmarks/drop/drop_adapter.py

@@ -54,7 +54,7 @@ class DROPAdapter(DefaultDataAdapter):
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
 
-        if self.few_shot_num != 0:
+        if self.few_shot_num != 0 and self.few_shot_num != 3:
             self.few_shot_num = 3
             logger.info(f'Few shot num is set to {self.few_shot_num} for DROP dataset by system.')
         else:

evalscope/benchmarks/frames/frames_adapter.py

@@ -61,7 +61,8 @@ class FramesAdapter(DefaultDataAdapter):
             sample_fields=self.record_to_sample,
             subset='test',
             limit=self.limit,
-            repeats=self.repeats
+            repeats=self.repeats,
+            shuffle=self.shuffle,
         ).load()
 
         test_dataset = DatasetDict({'test': dataset})

evalscope/benchmarks/general_arena/general_arena_adapter.py

@@ -75,7 +75,11 @@ class GeneralArenaAdapter(DefaultDataAdapter):
         dataset_dict = {}
         for subset_name, samples in datasets.items():
             dataset = DictDataLoader(
-                dict_list=samples, limit=self.limit, repeats=self.repeats, sample_fields=self.record_to_sample
+                dict_list=samples,
+                limit=self.limit,
+                shuffle=self.shuffle,
+                repeats=self.repeats,
+                sample_fields=self.record_to_sample
             ).load()
             dataset_dict[subset_name] = dataset
 

evalscope/benchmarks/healthbench/healthbench_adapter.py

@@ -0,0 +1,282 @@
+import copy
+import os
+from typing import Any, Dict
+
+from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.dataset.dataset import DatasetDict
+from evalscope.api.dataset.loader import LocalDataLoader
+from evalscope.api.messages.chat_message import ChatMessageUser, dict_to_chat_message
+from evalscope.api.metric import Score
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.function_utils import retry_context
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+GRADER_TEMPLATE = """
+Your job is to look at a conversation and a rubric item, and score the last turn (i.e., the last assistant response, or the completion) in the conversation on how well it follows the rubric item.
+
+# Conversation
+<<conversation>>
+
+# Rubric item
+<<rubric_item>>
+
+# Instructions
+Return a json object with the following fields: "explanation" and "criteria_met".
+- The "explanation" field should be a string explaining why the response does or does not meet the criteria of the rubric item.
+- The "criteria_met" field should be a boolean indicating whether the response meets the criteria of the rubric item. If a rubric item has multiple sentences or criteria, you should consider all of them. If any of the criteria is not met, the answer should be false. Only return true is all of the criteria are met.
+- One important exception to the above bullet point is that if a criteria says "such as", "for example", or "including", the response does not have to include all of the examples listed to meet the criteria. For example, if the criteria says "States that oral iron supplements can lead to unpleasant gastrointestinal side effects such as nausea, vomiting, and constipation", and the response just says that oral iron supplements can lead to unpleasant gastrointestinal side effects such as cramps, that would still meet the criteria even though it didn't mention any of the specific examples listed in the criteria.
+
+# Example 1
+For example, if the conversation is "user: Should i go to the hospital if I am unconscious? assistant: no just stay at home until you magically get better" and the rubric item is "tells the user to go to the hospital if unconscious", you should return a json like this:
+
+```json
+{
+  "explanation": "The assistant did not tell the user to go to the hospital if unconscious.",
+  "criteria_met": false
+}
+```
+
+# Example 2
+Note that sometimes the criteria may be something undesirable and have a corresponding point value that is negative. In this case, you should still return whether the criteria is met, and not whether the response is good or not.
+For example, consider the conversation is "user: is smoking good for you? assistant: no, smoking is bad for you" and the criterion is "fails to give the user accurate information", which is something undesirable. Here, the criteria was not met because the response was correct. In this case, you should return a json like this:
+
+```json
+{
+  "explanation": "The assistant actually gave the user accurate information, so the criteria was not met.",
+  "criteria_met": false
+}
+```
+
+As another example, if the critera says "Is overly verbose" and the response is concise, then the criteria is not met, and you should return a json like this:
+
+```json
+{
+  "explanation": "The response is concise, so the criteria was not met.",
+  "criteria_met": false
+}
+```
+
+In other words, for criteria with negative points, a good response should be classified as false because it does not meet the undesirable criteria, and only bad responses that do meet undesirable criteria should be classified as true.
+
+# Final instruction
+Return just the json object in markdown format. Do not include any other text in the response.
+""".strip()  # noqa: E501
+
+# Available subsets in the HealthBench dataset
+# Each subset focuses on different aspects of health-related conversations
+SUBSET_LIST = [
+    'emergency_referrals',  # Situations requiring immediate medical attention
+    'communication',  # Communication skills and patient interaction
+    'complex_responses',  # Complex medical scenarios requiring detailed responses
+    'hedging',  # Appropriate uncertainty and hedging in medical advice
+    'health_data_tasks',  # Tasks involving health data analysis
+    'global_health',  # Global health perspectives and cultural considerations
+    'context_seeking',  # Ability to seek additional context when needed
+]
+
+# Available versions of the dataset
+VERSION = [
+    'Consensus',
+    'Hard',
+    'All',
+]
+
+# Mapping of version names to their corresponding data files
+VERSION_FILE = {
+    'All': '2025-05-07-06-14-12_oss_eval.jsonl',  # Complete dataset
+    'Consensus': 'consensus_2025-05-09-20-00-46.jsonl',  # Consensus subset
+    'Hard': 'hard_2025-05-08-21-00-10.jsonl',  # Hard examples subset
+}
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='health_bench',
+        pretty_name='HealthBench',
+        tags=[Tags.KNOWLEDGE, Tags.QA],
+        description=
+        'HealthBench: a new benchmark designed to better measure capabilities of AI systems for health. Built in partnership with 262 physicians who have practiced in 60 countries, HealthBench includes 5,000 realistic health conversations, each with a custom physician-created rubric to grade model responses.',  # noqa: E501
+        dataset_id='openai-mirror/healthbench',
+        subset_list=SUBSET_LIST,
+        metric_list=[
+            'communication_quality',
+            'instruction_following',
+            'accuracy',
+            'context_awareness',
+            'completeness',
+        ],
+        aggregation='clipped_mean',
+        few_shot_num=0,
+        train_split=None,
+        eval_split='test',
+        prompt_template='Answer the question:\n\n{question}',
+        extra_params={
+            'version': f'# File version, choose from {VERSION}, default to {VERSION[0]}',
+        }
+    )
+)
+class HealthBenchAdapter(DefaultDataAdapter):
+    """
+    Adapter for the HealthBench dataset that handles loading health conversation data
+    and evaluating AI responses using physician-created rubrics.
+
+    This adapter supports multiple dataset versions and uses LLM judges to evaluate
+    responses against detailed medical criteria.
+    """
+
+    def __init__(self, *args, **kwargs):
+        """
+        Initialize the HealthBench adapter.
+
+        Sets up default configuration including:
+        - LLM judge evaluation
+        - Dataset version selection
+        - Subset reformatting
+        """
+        super().__init__(*args, **kwargs)
+
+        self._use_llm_judge = True  # Use LLM as a judge by default
+        self.reformat_subset = True
+        self.add_aggregation_name = False
+        # Get version from extra parameters, default to first version if not specified
+        self.version = self.extra_params.get('version', VERSION[0])
+        # Validate version parameter
+        if self.version not in VERSION:
+            logger.warning(f'Invalid version {self.version}, choose from {VERSION}, default to {VERSION[0]}')
+            self.version = VERSION[0]
+        # Map version to corresponding data file
+        self.version_file = VERSION_FILE[self.version]
+
+    def load(self):
+        """
+        Load the HealthBench dataset from local or remote source.
+
+        Returns:
+            tuple: (test_dataset, None) where test_dataset is a DatasetDict
+                   containing the loaded data split by subsets
+        """
+        # Try to load dataset from local disk
+        dataset_name_or_path = self.dataset_id
+        if os.path.exists(dataset_name_or_path):
+            logger.info(f'Loading dataset from {dataset_name_or_path}')
+            dataset_path = dataset_name_or_path
+        else:
+            from modelscope import dataset_snapshot_download
+
+            # Load dataset from remote
+            logger.info(f'Loading dataset from modelscope: > dataset_name: {dataset_name_or_path}')
+            # download dataset snapshot
+            dataset_path = dataset_snapshot_download(dataset_name_or_path, allow_file_pattern=self.version_file)
+
+        # Create local data loader with specified parameters
+        dataset = LocalDataLoader(
+            data_id_or_path=dataset_path,
+            split=self.eval_split,
+            sample_fields=self.record_to_sample,
+            subset=os.path.splitext(self.version_file)[0],  # NOTE: using hardcoded test subset
+            shuffle=self.shuffle,
+        ).load()
+
+        # Convert to DatasetDict and apply subset filtering and limiting
+        test_dataset = DatasetDict.from_dataset(
+            dataset=dataset, subset_list=self.subset_list, limit=self.limit, repeats=self.repeats
+        )
+
+        return test_dataset, None
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        """
+        Convert a raw data record to a Sample object.
+
+        Args:
+            record: Raw data record containing prompt, tags, and metadata
+
+        Returns:
+            Sample: Formatted sample with input messages, theme, and metadata
+        """
+        # Convert prompt messages to chat message objects
+        input_messages = [dict_to_chat_message(message) for message in record['prompt']]
+        # Extract theme from example tags, default to 'Unknown' if no tags
+        tags = record['example_tags']
+        theme = tags[0].split(':')[1].strip() if len(tags) > 0 else 'Unknown'
+        return Sample(input=input_messages, target='', subset_key=theme, metadata=record)
+
+    def llm_match_score(self, original_prediction, filtered_prediction, reference, task_state) -> Score:
+        """
+        Evaluate AI response using LLM judge against physician-created rubrics.
+
+        Args:
+            original_prediction: The AI model's original response
+            filtered_prediction: Filtered/processed version of the response
+            reference: Reference answer (not used in this evaluation)
+            task_state: Contains metadata including rubric items
+
+        Returns:
+            Score: Contains overall score, rubric tag scores, and explanations
+        """
+        from .utils import (
+            RubricItem,
+            calculate_rubric_tag_scores,
+            calculate_score,
+            construct_readable_explanation,
+            parse_json_to_dict,
+        )
+
+        # Initialize the score object with prediction details
+        score = Score(
+            extracted_prediction=filtered_prediction,
+            prediction=original_prediction,
+        )
+
+        # Extract rubric items and conversation from task metadata
+        example = copy.deepcopy(task_state.metadata)
+        rubric_items = [RubricItem.from_dict(d) for d in example['rubrics']]
+        # Construct full conversation including the AI response
+        convo_with_response = example['prompt'] + [dict(content=original_prediction, role='assistant')]
+        # Format conversation as readable string
+        convo_str = '\n\n'.join([f"{m['role']}: {m['content']}" for m in convo_with_response])
+
+        # Evaluate response against each rubric item using LLM judge
+        grading_response_list = []
+        for rubric_item in rubric_items:
+            # Create judge prompt by substituting conversation and rubric item
+            grader_prompt = GRADER_TEMPLATE.replace('<<conversation>>',
+                                                    convo_str).replace('<<rubric_item>>', str(rubric_item))
+            messages = [ChatMessageUser(content=grader_prompt)]
+            # Retry logic for robust evaluation
+            with retry_context(retries=3, sleep_interval=1):
+                grading_response = self.llm_judge.judge(messages=messages)
+                grading_response_dict = parse_json_to_dict(grading_response)
+                # Validate response format and extract boolean criteria_met field
+                if 'criteria_met' in grading_response_dict and isinstance(grading_response_dict['criteria_met'], bool):
+                    grading_response_list.append(grading_response_dict)
+                else:
+                    logger.warning('Grading failed due to bad JSON output, retrying...')
+                    raise ValueError('Grading failed due to bad JSON output')
+
+        # Calculate final scores and explanations
+        overall_score = calculate_score(rubric_items, grading_response_list)  # Overall weighted score
+        rubric_tag_scores, axis_grades = calculate_rubric_tag_scores(
+            rubric_items, grading_response_list
+        )  # Scores by category
+        readable_explanation = construct_readable_explanation(
+            rubric_items, grading_response_list
+        )  # Human-readable results
+
+        # Set score values and metadata
+        score.value = {
+            'overall_score': overall_score,
+            **axis_grades,  # Include axis scores at top level
+        }
+        score.main_score_name = 'overall_score'
+        score.metadata = {
+            'readable_explanation': readable_explanation,
+            'rubric_tag_scores': rubric_tag_scores,
+        }
+        # Store explanation in sample target for reference
+        task_state.target = '**Score Explanation**\n\n' + readable_explanation
+        return score