evalscope 0.17.1__py3-none-any.whl → 1.0.1__py3-none-any.whl

evalscope/benchmarks/bbh/bbh_adapter.py

@@ -1,17 +1,16 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 
-import json
 import os
-import random
 import re
+from typing import Any, Dict
 
-from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.constants import AnswerKeys
-from evalscope.metrics import exact_match
+from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.evaluator import TaskState
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
 from evalscope.utils.logger import get_logger
 
-# flake8: noqa
-
 logger = get_logger()
 
 # BBH multiple choice subset list
@@ -55,160 +54,89 @@ FREE_FORM_LIST = [
 TASK_TYPE = 'task_type'
 SUBSET_LIST = MULTIPLE_CHOICE_LIST + FREE_FORM_LIST
 
-
-@Benchmark.register(
-    name='bbh',
-    pretty_name='BBH',
-    tags=['Reasoning'],
-    description=
-    'The BBH (Big Bench Hard) benchmark is a collection of challenging tasks designed to evaluate the reasoning capabilities of AI models. It includes both free-form and multiple-choice tasks, covering a wide range of reasoning skills.',  # noqa: E501
-    dataset_id='modelscope/bbh',
-    subset_list=SUBSET_LIST,
-    metric_list=['AverageAccuracy'],
-    few_shot_num=3,
-    train_split=None,
-    eval_split='test',
-    prompt_template="Q: {query}\nA: Let's think step by step.",
+PROMPT_TEMPLATE = """
+Q: {question}
+A: Let's think step by step. Put your final answer in the format of "So the answer is $ANSWER" (without quotes and markdown) where $ANSWER is the answer to the problem.
+""".lstrip()  # noqa: E501
+
+FEWSHOT_TEMPLATE = """
+{fewshot}
+
+""".lstrip() + PROMPT_TEMPLATE
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='bbh',
+        pretty_name='BBH',
+        dataset_id='evalscope/bbh',
+        tags=[Tags.REASONING],
+        description=
+        'The BBH (Big Bench Hard) benchmark is a collection of challenging tasks designed to evaluate the reasoning capabilities of AI models. It includes both free-form and multiple-choice tasks, covering a wide range of reasoning skills.',  # noqa: E501
+        subset_list=SUBSET_LIST,
+        few_shot_num=3,
+        train_split=None,
+        eval_split='test',
+        metric_list=['acc'],
+        prompt_template=PROMPT_TEMPLATE,
+        few_shot_prompt_template=FEWSHOT_TEMPLATE,
+    )
 )
-class BBHAdapter(DataAdapter):
+class BBHAdapter(DefaultDataAdapter):
     """
     Adapter for BBH free-form and multiple-choices sub-tasks.
     """
 
     def __init__(self, **kwargs):
-
         few_shot_num = kwargs.get('few_shot_num', 3)
 
         if few_shot_num != 3 and few_shot_num != 0:
-            logger.error(f'BBH uses 3-shot examples with CoT or 0-shot by system, but got {few_shot_num}. '
-                         f'Use 3-shot by default.')
+            logger.error(
+                f'BBH uses 3-shot examples with CoT or 0-shot by system, but got {few_shot_num}. '
+                f'Use 3-shot by default.'
+            )
             kwargs['few_shot_num'] = 3
 
         super().__init__(**kwargs)
 
-    def load_from_disk(self, dataset_name_or_path, subset_list, work_dir, **kwargs) -> dict:
-        data_dict = {}
-        for subset_name in subset_list:
-            for split_name in [self.eval_split]:
-                if os.path.exists(dataset_name_or_path):
-                    file_path = os.path.join(dataset_name_or_path, f'{subset_name}.json')
-                else:
-                    file_path: str = os.path.join(work_dir, dataset_name_or_path, f'{subset_name}.json')
-                if os.path.exists(file_path):
-                    with open(file_path, 'r', encoding='utf-8') as f:
-                        examples = json.load(f)['examples']
-                        if subset_name in data_dict:
-                            data_dict[subset_name].update({split_name: examples})
-                        else:
-                            data_dict[subset_name] = {split_name: examples}
-
-        return data_dict
-
-    def gen_prompt(self, input_d: dict, few_shot_list: list, **kwargs) -> dict:
-        """
-        Generate model prompt from raw data, unify the prompt format for bbh(multiple choice) benchmark.
-
-        Args:
-            input_d (dict): The raw input. A single data format of the BBH:
-
-            {
-                'input': '((-1 + 2 + 9 * 5) - (-2 + -4 + -4 * -7)) =',
-                'target': '24',
-            }
-
-        Returns:
-            {'data': ['xxx']}
-        """
-        # few_shot_list: should be ['xxxx']
-        if len(few_shot_list) > 0:
-            cot_prompts = 'Follow the given examples and answer the question.\n' + few_shot_list[0]
-        else:
-            cot_prompts = ''
-        full_prompt = cot_prompts + self.prompt_template.format(query=input_d['input'])
-
-        return self.gen_prompt_data(full_prompt)
-
-    def gen_prompts(self, data_dict: dict) -> dict:
-        """
-        Generate dataset prompts from raw input, unify the prompt format for different datasets.
-
-        Args:
-            data_dict:  Refer to the output of load method: evalscope.benchmarks.benchmark.Benchmark.load
-
-        Returns:
-            {'subset_name': [prompt_d_1, prompt_d_2, ...]}
-            prompt_d_i (dict): refer to the output of gen_prompt method.
-
-        e.g. train -- few-shot data, test -- target dataset to evaluate.
-        """
-        res_dict: dict = {}
-
-        if self.few_shot_num < 0:
-            raise ValueError(f'Invalid shot_num: {self.few_shot_num} for few-shot evaluation.')
-
-        logger.info(f'Use default settings: '
-                    f'> few_shot_num: {self.few_shot_num}, '
-                    f'> few_shot_split: {self.train_split}, '
-                    f'> target_eval_split: {self.eval_split}')
-
-        for sub_name, sub_data_dict in data_dict.items():
-            few_shot_data = []
-            if self.few_shot_num > 0:
-                with open(
-                        os.path.join(os.path.dirname(__file__), 'cot_prompts', f'{sub_name}.txt'), 'r',
-                        encoding='utf-8') as f:
-                    cot_prompt_str = f.read()
-                few_shot_data = [cot_prompt_str]
-
-            res_dict[sub_name] = []
-            for sample_d in sub_data_dict[self.eval_split]:
-                prompt_d = self.gen_prompt(input_d=sample_d, few_shot_list=few_shot_data)
-                sample_d_new = sample_d.copy()
-                if sub_name in MULTIPLE_CHOICE_LIST:
-                    sample_d_new[TASK_TYPE] = MULTIPLE_CHOICE
-                elif sub_name in FREE_FORM_LIST:
-                    sample_d_new[TASK_TYPE] = FREE_FORM
-                else:
-                    raise ValueError(f'Invalid subset name: {sub_name}')
-
-                prompt_d[AnswerKeys.RAW_INPUT] = sample_d_new
-                res_dict[sub_name].append(prompt_d)
-
-        return res_dict
-
-    def get_gold_answer(self, input_d: dict) -> str:
-        # Get the gold choice
-        gold = input_d.get('target', '')
-        # remove brackets
-        if gold is None:
-            logger.error(f'BBHAdapter: gold is None.')
-        gold = gold.replace('(', '').replace(')', '')
-        return gold
-
-    def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = 'checkpoint') -> str:
-        """
-        Parse the model output to get the answer. Could be the best choice index.
-
-        Args:
-            result: Predicted answer from the model. Usually a string for chat.
-            raw_input_d (dict): The raw input. Depending on the dataset.
-            eval_type: 'checkpoint' or 'service' or `custom`, default: 'checkpoint'
-
-        Returns:
-            The parsed answer. Depending on the dataset. Usually a string for chat.
-        """
-        # Note: to use same extraction method for both of checkpoint/service/custom.
-        task_type: str = raw_input_d.get(TASK_TYPE)
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        input = record['input']
+        target = record['target'].replace('(', '').replace(')', '').strip()  # Clean up the target answer
+
+        # Determine task type based on subset name
+        task_type = None
+        subset_name = self.current_subset_name
+        if subset_name in MULTIPLE_CHOICE_LIST:
+            task_type = MULTIPLE_CHOICE
+        elif subset_name in FREE_FORM_LIST:
+            task_type = FREE_FORM
+
+        metadata = {TASK_TYPE: task_type}
+
+        return Sample(input=input, target=target, metadata=metadata, subset_key=subset_name)
+
+    def format_fewshot_template(self, fewshot: str, sample: Sample) -> str:
+        # Load CoT prompts from file for BBH
+        subset_name = sample.subset_key
+        if subset_name:
+            cot_file_path = os.path.join(os.path.dirname(__file__), 'cot_prompts', f'{subset_name}.txt')
+            if os.path.exists(cot_file_path):
+                with open(cot_file_path, 'r', encoding='utf-8') as f:
+                    fewshot = f.read().strip()
+        return self.few_shot_prompt_template.format(
+            fewshot=fewshot,
+            question=sample.input,
+        )
+
+    def extract_answer(self, prediction: str, task_state: TaskState):
+        task_type = task_state.metadata.get(TASK_TYPE)
 
         if task_type == MULTIPLE_CHOICE:
-            return self._extract_mc_answer(result)
+            return self._extract_mc_answer(prediction)
         elif task_type == FREE_FORM:
-            return self._extract_ff_answer(result)
+            return self._extract_ff_answer(prediction)
         else:
-            raise ValueError(f'Invalid task type: {task_type}')
-
-    def match(self, gold: str, pred: str) -> float:
-        return exact_match(gold=gold, pred=pred)
+            return prediction.strip()
 
     @classmethod
     def _extract_mc_answer(cls, ans: str) -> str:

evalscope/benchmarks/bfcl/bfcl_adapter.py

@@ -1,12 +1,17 @@
-import copy
-import importlib
 import json
 import re
 import traceback
-from typing import Any, List
-
-from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.constants import EvalType
+from typing import Any, Dict
+
+from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.evaluator import TaskState
+from evalscope.api.messages.chat_message import ChatMessageUser
+from evalscope.api.metric import Score
+from evalscope.api.model import Model, ModelOutput
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.import_utils import check_import
 from evalscope.utils.logger import get_logger
 
 logger = get_logger()
@@ -32,47 +37,43 @@ SUBJECT_MAPPING = {
 }
 
 
-@Benchmark.register(
-    name='bfcl_v3',
-    pretty_name='BFCL-v3',
-    tags=['Agent', 'Function Calling'],
-    description=
-    'Berkeley Function Calling Leaderboard (BFCL), the **first comprehensive and executable function call evaluation** '
-    'dedicated to assessing Large Language Models\' (LLMs) ability to invoke functions. Unlike previous evaluations, '
-    'BFCL accounts for various forms of function calls, diverse scenarios, and executability. '
-    'Need to run `pip install bfcl-eval` before evaluating. '
-    '[Usage Example](https://evalscope.readthedocs.io/zh-cn/latest/third_party/bfcl_v3.html)',  # noqa: E501
-    dataset_id='AI-ModelScope/bfcl_v3',
-    subset_list=list(SUBJECT_MAPPING.keys()),
-    model_adapter='bfcl_server',
-    metric_list=['AverageAccuracy'],
-    few_shot_num=0,
-    train_split=None,
-    eval_split='train',
-    extra_params={
-        'underscore_to_dot': True,
-        'is_fc_model': True,
-    })
-class BFCLAdapter(DataAdapter):
+@register_benchmark(
+    BenchmarkMeta(
+        name='bfcl_v3',
+        pretty_name='BFCL-v3',
+        tags=[Tags.FUNCTION_CALLING],
+        description='Berkeley Function Calling Leaderboard (BFCL), the **first comprehensive '
+        'and executable function call evaluation** '
+        'dedicated to assessing Large Language Models\' (LLMs) ability to invoke '
+        'functions. Unlike previous evaluations, '
+        'BFCL accounts for various forms of function calls, diverse scenarios, and executability. '
+        'Need to run `pip install bfcl-eval==2025.6.16` before evaluating. '
+        '[Usage Example](https://evalscope.readthedocs.io/zh-cn/latest/third_party/bfcl_v3.html)',
+        dataset_id='AI-ModelScope/bfcl_v3',
+        subset_list=list(SUBJECT_MAPPING.keys()),
+        metric_list=['acc'],
+        eval_split='train',
+        extra_params={
+            'underscore_to_dot': True,
+            'is_fc_model': True,
+        }
+    )
+)
+class BFCLAdapter(DefaultDataAdapter):
+    """
+    BFCL adapter using the new data processing framework.
+    """
 
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
 
-        spec = importlib.util.find_spec('bfcl_eval')
-        if spec is None:
-            raise ImportError(
-                '`bfcl_eval` not found, please install it with `pip install bfcl-eval` before evaluating.')
+        check_import('bfcl_eval', package='bfcl-eval==2025.6.16', raise_error=True)
 
         self.category_map = SUBJECT_MAPPING
+        self.reformat_subset = True
 
-        extra_params = kwargs.get('extra_params', {})
-        self.underscore_to_dot = extra_params.get('underscore_to_dot', False)
-        self.is_fc_model = extra_params.get('is_fc_model', True)
-
-    def load(self, **kwargs):
-        kwargs['subset_list'] = ['default']
-        data_dict = super().load(**kwargs)
-        return self.reformat_subset(data_dict, subset_key='subset', format='{}')
+        self.underscore_to_dot = self.extra_params.get('underscore_to_dot', True)
+        self.is_fc_model = self.extra_params.get('is_fc_model', True)
 
     def preprocess_row(self, row: dict):
         """
@@ -87,151 +88,167 @@ class BFCLAdapter(DataAdapter):
         row['initial_config'] = json.loads(row['initial_config'])
         row['is_fc_model'] = self.is_fc_model
 
-    def gen_prompt(self, input_d, subset_name, few_shot_list, **kwargs):
-        self.preprocess_row(input_d)
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        """Convert a data record to a Sample object."""
+        self.preprocess_row(record)
 
         # If the model is a function calling model, we need to remove the system prompt
         if self.is_fc_model:
-            turns = input_d['turns']
+            turns = record['turns']
             new_turns = []
             for turn_idx, messages in enumerate(turns):
                 current_messages = messages.copy()
                 if len(current_messages) > 0 and current_messages[0]['role'] == 'system':
                     current_messages = current_messages[1:]
                 new_turns.append(current_messages)
-            input_d['turns'] = new_turns
-
-        return self.gen_prompt_data(prompt='', messages=input_d)
-
-    def get_gold_answer(self, input_d: dict) -> str:
-        # Get the gold choice
-        return input_d.get('ground_truth', )
-
-    def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = EvalType.CHECKPOINT) -> dict:
-        row = copy.deepcopy(raw_input_d)
-        del row['turns']  # Remove turns as they are not needed for the match function
-
-        row['generation'] = result
-        return row
-
-    def match(self, gold: dict, pred: dict) -> dict:
+            record['turns'] = new_turns
+
+        return Sample(
+            input=[ChatMessageUser(content='')],
+            target='',  # Will use the record for evaluation
+            subset_key=record['subset'],
+            metadata=record  # Store the full record for evaluation
+        )
+
+    def _on_inference(self, model: Model, sample: Sample) -> ModelOutput:
+        from .generation import predict
+        return predict(model, sample)
+
+    def match_score(
+        self, original_prediction: str, filtered_prediction: str, reference: str, task_state: TaskState
+    ) -> Score:
         from bfcl_eval.eval_checker.ast_eval.ast_checker import ast_checker
         from bfcl_eval.eval_checker.multi_turn_eval.multi_turn_checker import multi_turn_checker
-        from bfcl_eval.model_handler.utils import (convert_to_function_call, default_decode_ast_prompting,
-                                                   default_decode_execute_prompting)
+        from bfcl_eval.model_handler.utils import (
+            convert_to_function_call,
+            default_decode_ast_prompting,
+            default_decode_execute_prompting,
+        )
         from bfcl_eval.utils import is_empty_output
 
-        # NOTE: This is hardcoded dummy model since its only use is to infer underscore_to_dot
-        # which decides if model was provided with functions of the type
-        # spotify.list_songs or spotify_list_songs
-        # It is False for all llama models (when using via prompting)
-        # and True for API calls
-        if self.underscore_to_dot:
-            dummy_model = 'gpt-4o-2024-11-20-FC'
-        else:
-            dummy_model = 'meta-llama/Llama-3.3-70B-Instruct-FC'
-
-        row = pred
-        test_category = re.sub(r'_[0-9_-]+$', '', row['id'])
-        if test_category in {'irrelevance', 'live_irrelevance', 'live_relevance'}:
-            error = None
-            try:
-                if self.is_fc_model:
-                    decoded_tool_calls = []
-                    for tool_call in row['generation'][0]:
-                        name = list(tool_call.keys())[0]
-                        params = json.loads(tool_call[name])
-                        decoded_tool_calls.append({name: params})
-                else:
-                    decoded_tool_calls = default_decode_ast_prompting(row['generation'][0][0], row['language'])
-
-                # successful decode means valid function call was present
-                contains_func_call = True
-                if is_empty_output(decoded_tool_calls):
-                    # Empty output is not considered as a valid function call
+        score = Score(
+            extracted_prediction=filtered_prediction,
+            prediction=original_prediction,
+        )
+
+        try:
+            # NOTE: This is hardcoded dummy model since its only use is to infer underscore_to_dot
+            if self.underscore_to_dot:
+                dummy_model = 'gpt-4o-2024-11-20-FC'
+            else:
+                dummy_model = 'meta-llama/Llama-3.3-70B-Instruct-FC'
+
+            row = task_state.metadata
+            test_category = re.sub(r'_[0-9_-]+$', '', row['id'])
+
+            if test_category in {'irrelevance', 'live_irrelevance', 'live_relevance'}:
+                error = None
+                try:
+                    if self.is_fc_model:
+                        decoded_tool_calls = []
+                        for tool_call in row['generation'][0]:
+                            name = list(tool_call.keys())[0]
+                            params = tool_call[name]
+                            decoded_tool_calls.append({name: params})
+                    else:
+                        decoded_tool_calls = default_decode_ast_prompting(row['generation'][0][0], row['language'])
+
+                    # successful decode means valid function call was present
+                    contains_func_call = True
+                    if is_empty_output(decoded_tool_calls):
+                        # Empty output is not considered as a valid function call
+                        contains_func_call = False
+                        error = 'Empty decoded output.'
+                except Exception:
                     contains_func_call = False
-                    error = 'Empty decoded output.'
-            except Exception:
-                contains_func_call = False
-                error = f'Failed to decode with traceback: {traceback.format_exc()}'
-            finally:
-                valid = contains_func_call if test_category == 'live_relevance' else not contains_func_call
-                score_result = {'valid': valid, 'error_message': error}
-
-        elif row['multi_turn']:
-            # each step might give a list of tool calls and each turn is multi-step
-            # and multi-turn has generations of all the turns
-            # hence in a multi-turn setting,
-            # multi_turn_decoded_generations is a list of list of list of strings
-            multi_turn_decoded_generations: list[list[list[str]]] = []
-            for single_turn_generations in row['generation']:
-                single_turn_decoded_generations: list[list[str]] = []
-                for generation in single_turn_generations:
-                    try:
-                        if self.is_fc_model:
-                            tool_calls = convert_to_function_call(generation)
-                        else:
-                            tool_calls = default_decode_execute_prompting(generation)
-
-                        single_turn_decoded_generations.append(tool_calls)
-                    except Exception:
-                        single_turn_decoded_generations.append([generation])
-
-                multi_turn_decoded_generations.append(single_turn_decoded_generations)
-
-            try:
-                raw_score_result = multi_turn_checker(
-                    multi_turn_decoded_generations,
-                    row['ground_truth'],
-                    row,
-                    test_category,
-                    dummy_model,
-                )
-            except Exception:
-                raw_score_result = {
-                    'valid': False,
-                    'error_type': 'multi_turn:checker_failed',
-                    'error_message': f'Failed to grade multi-turn. Traceback: {traceback.format_exc()}',
-                }
+                    error = f'Failed to decode with traceback: {traceback.format_exc()}'
+                finally:
+                    valid = contains_func_call if test_category == 'live_relevance' else not contains_func_call
+                    score_result = {'valid': valid, 'error_message': error}
+
+            elif row['multi_turn']:
+                # each step might give a list of tool calls and each turn is multi-step
+                # and multi-turn has generations of all the turns
+                # hence in a multi-turn setting,
+                # multi_turn_decoded_generations is a list of list of list of strings
+                multi_turn_decoded_generations: list[list[list[str]]] = []
+                for single_turn_generations in row['generation']:
+                    single_turn_decoded_generations: list[list[str]] = []
+                    for generation in single_turn_generations:
+                        try:
+                            if self.is_fc_model:
+                                tool_calls = convert_to_function_call(generation)
+                            else:
+                                tool_calls = default_decode_execute_prompting(generation)
+
+                            single_turn_decoded_generations.append(tool_calls)
+                        except Exception:
+                            single_turn_decoded_generations.append([generation])
+
+                    multi_turn_decoded_generations.append(single_turn_decoded_generations)
+
+                try:
+                    raw_score_result = multi_turn_checker(
+                        multi_turn_decoded_generations,
+                        row['ground_truth'],
+                        row,
+                        test_category,
+                        dummy_model,
+                    )
+                except Exception:
+                    raw_score_result = {
+                        'valid': False,
+                        'error_type': 'multi_turn:checker_failed',
+                        'error_message': f'Failed to grade multi-turn. Traceback: {traceback.format_exc()}',
+                    }
 
-            score_result = {
-                'valid': float(raw_score_result['valid']),
-                'error_message': raw_score_result.get('error_message', ''),
-                'error_type': raw_score_result.get('error_type', ''),
-            }
-        else:
-            try:
-                if self.is_fc_model:
-                    decoded_tool_calls = []
-                    for tool_call in row['generation'][0]:
-                        name = list(tool_call.keys())[0]
-                        params = json.loads(tool_call[name])
-                        decoded_tool_calls.append({name: params})
-                else:
-                    decoded_tool_calls = default_decode_ast_prompting(row['generation'][0][0], row['language'])
-
-                score_result = ast_checker(
-                    row['functions'],
-                    decoded_tool_calls,
-                    row['ground_truth'],
-                    row['language'],
-                    row['test_category'],
-                    dummy_model,
-                )
-            except Exception:
                 score_result = {
-                    'valid': False,
-                    'error_message': f'Invalid syntax. Failed to decode AST. Traceback: {traceback.format_exc()}',
-                    'error_type': 'ast_decoder:decoder_failed',
+                    'valid': float(raw_score_result['valid']),
+                    'error_message': raw_score_result.get('error_message', ''),
+                    'error_type': raw_score_result.get('error_type', ''),
                 }
-
-        return {
-            'AverageAccuracy': float(score_result['valid']),
-            'raw_score_result': score_result,
-        }
-
-    def compute_metric(self, review_res_list: List[dict], **kwargs) -> Any:
-        # aggregate review results
-        res_dict = super().compute_dict_metric(review_res_list, **kwargs)
-
-        return super().compute_metric(res_dict, **kwargs)
+            else:
+                try:
+                    if self.is_fc_model:
+                        decoded_tool_calls = []
+                        for tool_call in row['generation'][0]:
+                            name = list(tool_call.keys())[0]
+                            params = tool_call[name]
+                            decoded_tool_calls.append({name: params})
+                    else:
+                        decoded_tool_calls = default_decode_ast_prompting(row['generation'][0][0], row['language'])
+
+                    score_result = ast_checker(
+                        row['functions'],
+                        decoded_tool_calls,
+                        row['ground_truth'],
+                        row['language'],
+                        row['test_category'],
+                        dummy_model,
+                    )
+                except Exception:
+                    score_result = {
+                        'valid': False,
+                        'error_message': f'Invalid syntax. Failed to decode AST. Traceback: {traceback.format_exc()}',
+                        'error_type': 'ast_decoder:decoder_failed',
+                    }
+
+            score.value = {
+                'acc': float(score_result['valid']),
+            }
+            score.explanation = score_result.get('error_message', 'Evaluation completed')
+            score.metadata = {
+                'raw_score_result': score_result,
+                'test_category': test_category,
+                'underscore_to_dot': self.underscore_to_dot,
+                'is_fc_model': self.is_fc_model
+            }
+            score.main_score_name = 'acc'
+
+        except Exception:
+            logger.error(f'Evaluation failed for sample: {task_state.sample_id}\n{traceback.format_exc()}')
+            score.value = {'acc': 0.0}
+            score.explanation = 'Evaluation failed with an unexpected error.'
+            score.metadata = {'error': traceback.format_exc()}
+            score.main_score_name = 'acc'
+        return score