evalscope 1.0.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

evalscope/benchmarks/bfcl/{bfcl_adapter.py → v3/bfcl_v3_adapter.py}

@@ -1,10 +1,9 @@
-import importlib
 import json
 import re
 import traceback
-from typing import Any, Dict
+from typing import Any, Dict, List
 
-from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter
+from evalscope.api.benchmark import AgentAdapter, BenchmarkMeta
 from evalscope.api.dataset import Sample
 from evalscope.api.evaluator import TaskState
 from evalscope.api.messages.chat_message import ChatMessageUser
@@ -12,6 +11,8 @@ from evalscope.api.metric import Score
 from evalscope.api.model import Model, ModelOutput
 from evalscope.api.registry import register_benchmark
 from evalscope.constants import Tags
+from evalscope.report import Category, Report, Subset, unweighted_average_from_subsets, weighted_average_from_subsets
+from evalscope.utils.import_utils import check_import
 from evalscope.utils.logger import get_logger
 
 logger = get_logger()
@@ -36,19 +37,25 @@ SUBJECT_MAPPING = {
     'multi_turn_long_context': 'MULTI_TURN'
 }
 
+BFCL_V3_TO_V4_SUBJECT_MAPPING = {
+    'simple': 'simple_python',
+    'java': 'simple_java',
+    'javascript': 'simple_javascript',
+}
+
 
 @register_benchmark(
     BenchmarkMeta(
         name='bfcl_v3',
         pretty_name='BFCL-v3',
-        tags=[Tags.FUNCTION_CALLING],
+        tags=[Tags.FUNCTION_CALLING, Tags.AGENT],
         description='Berkeley Function Calling Leaderboard (BFCL), the **first comprehensive '
         'and executable function call evaluation** '
         'dedicated to assessing Large Language Models\' (LLMs) ability to invoke '
         'functions. Unlike previous evaluations, '
         'BFCL accounts for various forms of function calls, diverse scenarios, and executability. '
-        'Need to run `pip install bfcl-eval==2025.6.16` before evaluating. '
-        '[Usage Example](https://evalscope.readthedocs.io/zh-cn/latest/third_party/bfcl_v3.html)',
+        'Need to run `pip install bfcl-eval==2025.10.27.1` before evaluating. '
+        '[Usage Example](https://evalscope.readthedocs.io/en/latest/third_party/bfcl_v3.html)',
         dataset_id='AI-ModelScope/bfcl_v3',
         subset_list=list(SUBJECT_MAPPING.keys()),
         metric_list=['acc'],
@@ -59,7 +66,7 @@ SUBJECT_MAPPING = {
         }
     )
 )
-class BFCLAdapter(DefaultDataAdapter):
+class BFCLV3Adapter(AgentAdapter):
     """
     BFCL adapter using the new data processing framework.
     """
@@ -67,14 +74,12 @@ class BFCLAdapter(DefaultDataAdapter):
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
 
-        spec = importlib.util.find_spec('bfcl_eval')
-        if spec is None:
-            raise ImportError(
-                '`bfcl_eval` not found, please install it with `pip install bfcl-eval==2025.6.16` before evaluating.'
-            )
+        check_import('bfcl_eval', package='bfcl-eval==2025.10.27.1', raise_error=True, feature_name=self.pretty_name)
 
         self.category_map = SUBJECT_MAPPING
         self.reformat_subset = True
+        self.add_overall_metric = False
+        self.add_aggregation_name = False
 
         self.underscore_to_dot = self.extra_params.get('underscore_to_dot', True)
         self.is_fc_model = self.extra_params.get('is_fc_model', True)
@@ -108,8 +113,8 @@ class BFCLAdapter(DefaultDataAdapter):
             record['turns'] = new_turns
 
         return Sample(
-            input=[ChatMessageUser(content='')],
-            target='',  # Will use the record for evaluation
+            input=[ChatMessageUser(content=json.dumps(record['turns']))],
+            target=json.dumps(record['ground_truth']),  # Will use the record for evaluation
             subset_key=record['subset'],
             metadata=record  # Store the full record for evaluation
         )
@@ -130,6 +135,8 @@ class BFCLAdapter(DefaultDataAdapter):
         )
         from bfcl_eval.utils import is_empty_output
 
+        from .utils import convert_format_language, convert_language
+
         score = Score(
             extracted_prediction=filtered_prediction,
             prediction=original_prediction,
@@ -143,7 +150,7 @@ class BFCLAdapter(DefaultDataAdapter):
                 dummy_model = 'meta-llama/Llama-3.3-70B-Instruct-FC'
 
             row = task_state.metadata
-            test_category = re.sub(r'_[0-9_-]+$', '', row['id'])
+            test_category = BFCL_V3_TO_V4_SUBJECT_MAPPING.get(row['test_category'], row['test_category'])
 
             if test_category in {'irrelevance', 'live_irrelevance', 'live_relevance'}:
                 error = None
@@ -155,7 +162,9 @@ class BFCLAdapter(DefaultDataAdapter):
                             params = tool_call[name]
                             decoded_tool_calls.append({name: params})
                     else:
-                        decoded_tool_calls = default_decode_ast_prompting(row['generation'][0][0], row['language'])
+                        decoded_tool_calls = default_decode_ast_prompting(
+                            row['generation'][0][0], convert_format_language(row['language'])
+                        )
 
                     # successful decode means valid function call was present
                     contains_func_call = True
@@ -220,14 +229,16 @@ class BFCLAdapter(DefaultDataAdapter):
                             params = tool_call[name]
                             decoded_tool_calls.append({name: params})
                     else:
-                        decoded_tool_calls = default_decode_ast_prompting(row['generation'][0][0], row['language'])
+                        decoded_tool_calls = default_decode_ast_prompting(
+                            row['generation'][0][0], convert_format_language(row['language'])
+                        )
 
                     score_result = ast_checker(
                         row['functions'],
                         decoded_tool_calls,
                         row['ground_truth'],
-                        row['language'],
-                        row['test_category'],
+                        convert_language(row['language']),
+                        test_category,
                         dummy_model,
                     )
                 except Exception:
@@ -256,3 +267,104 @@ class BFCLAdapter(DefaultDataAdapter):
             score.metadata = {'error': traceback.format_exc()}
             score.main_score_name = 'acc'
         return score
+
+    def _on_generate_report_end(self, report: Report, output_dir, **kwargs):
+        """
+        Finalize the report generation process. Calculate the overall score.
+
+        Track the number of each category.
+        - step1: simple, java, javascript unweighted average as simple_ast
+        - step2.1: simple_ast, multiple, parallel, parallel_multiple unweighted average as ast_non_live
+        - step2.2: live_simple, live_multiple, live_parallel, live_parallel_multiple weighted average as ast_live
+        - step2.3: irrelevance as hallucination_non_live
+        - step2.4: live_irrelevance, live_relevance weighted average as hallucination_live
+        - step2.5: multi_turn_base as multi_turn_base
+        - step2.6: multi_turn_miss_func, multi_turn_miss_param, multi_turn_long_context weighted average as multi_turn_augmented
+        - step3.1: ast_non_live, hallucination_non_live unweighted average as non_live
+        - step3.2: ast_live, hallucination_live weighted average as live
+        - step3.3: multi_turn_base, multi_turn_augmented unweighted average as multi_turn
+        - step4: non_live, live, multi_turn unweighted average as overall
+        Args:
+            report (Report): The generated evaluation report.
+            output_dir (str): The directory to save the report.
+
+        Returns:
+            None
+        """  # noqa: E501
+        for metric in report.metrics:
+            # Collect all subsets in a dictionary for easy access
+            subset_dict: Dict[str, Subset] = {}
+            for category in metric.categories:
+                for subset in category.subsets:
+                    subset_dict[subset.name] = subset
+
+            # Step 1: Calculate simple_ast (simple, java, javascript unweighted average)
+            simple_subsets = ['simple', 'java', 'javascript']
+            simple_ast = unweighted_average_from_subsets(simple_subsets, subset_dict)
+            subset_dict['simple_ast'] = simple_ast
+
+            # Step 2.1: Calculate ast_non_live
+            # (simple_ast, multiple, parallel, parallel_multiple unweighted average)
+            ast_non_live_subsets = ['simple_ast', 'multiple', 'parallel', 'parallel_multiple']
+            ast_non_live = unweighted_average_from_subsets(ast_non_live_subsets, subset_dict)
+            subset_dict['ast_non_live'] = ast_non_live
+
+            # Step 2.2: Calculate ast_live
+            # (live_simple, live_multiple, live_parallel, live_parallel_multiple weighted average)
+            live_subsets = ['live_simple', 'live_multiple', 'live_parallel', 'live_parallel_multiple']
+            ast_live = weighted_average_from_subsets(live_subsets, subset_dict)
+            subset_dict['ast_live'] = ast_live
+
+            # Step 2.3: hallucination_non_live (irrelevance)
+            if 'irrelevance' in subset_dict:
+                subset_dict['hallucination_non_live'] = subset_dict['irrelevance']
+            else:
+                subset_dict['hallucination_non_live'] = Subset(name='hallucination_non_live', score=0, num=0)
+
+            # Step 2.4: Calculate hallucination_live (live_irrelevance, live_relevance weighted average)
+            hallucination_live_subsets = ['live_irrelevance', 'live_relevance']
+            hallucination_live = weighted_average_from_subsets(hallucination_live_subsets, subset_dict)
+            subset_dict['hallucination_live'] = hallucination_live
+
+            # Step 2.5: multi_turn_base
+            if 'multi_turn_base' not in subset_dict:
+                subset_dict['multi_turn_base'] = Subset(name='multi_turn_base', score=0, num=0)
+
+            # Step 2.6: Calculate multi_turn_augmented
+            # (multi_turn_miss_func, multi_turn_miss_param, multi_turn_long_context weighted average)
+            multi_turn_augmented_subsets = ['multi_turn_miss_func', 'multi_turn_miss_param', 'multi_turn_long_context']
+            multi_turn_augmented = weighted_average_from_subsets(multi_turn_augmented_subsets, subset_dict)
+            subset_dict['multi_turn_augmented'] = multi_turn_augmented
+
+            # Step 3.1: Calculate non_live (ast_non_live, hallucination_non_live unweighted average)
+            non_live_subsets = ['ast_non_live', 'hallucination_non_live']
+            non_live = unweighted_average_from_subsets(non_live_subsets, subset_dict)
+            subset_dict['non_live'] = non_live
+
+            # Step 3.2: Calculate live (ast_live, hallucination_live weighted average)
+            live_agg_subsets = ['ast_live', 'hallucination_live']
+            live = weighted_average_from_subsets(live_agg_subsets, subset_dict)
+            subset_dict['live'] = live
+
+            # Step 3.3: Calculate multi_turn (multi_turn_base, multi_turn_augmented unweighted average)
+            multi_turn_subsets = ['multi_turn_base', 'multi_turn_augmented']
+            multi_turn = unweighted_average_from_subsets(multi_turn_subsets, subset_dict)
+            subset_dict['multi_turn'] = multi_turn
+
+            # Step 4: Calculate overall (non_live, live, multi_turn unweighted average)
+            overall_subsets = ['non_live', 'live', 'multi_turn']
+            overall = unweighted_average_from_subsets(overall_subsets, subset_dict)
+            subset_dict['overall'] = overall
+
+            # Add computed scores to the category
+            computed_subset_names = ['non_live', 'live', 'multi_turn', 'overall']
+
+            # Add the computed scores as new subsets in the metric
+            dummy_subsets = []
+            for subset_name in computed_subset_names:
+                if subset_name in subset_dict:
+                    subset = subset_dict[subset_name]
+                    subset.name = subset_name.upper()
+                    dummy_subsets.append(subset)
+            dummy_category = Category(name='-', subsets=dummy_subsets)
+            metric.categories.append(dummy_category)

evalscope/benchmarks/bfcl/{generation.py → v3/generation.py}

@@ -72,13 +72,14 @@ def generate_turn(model: Model, row: dict[str, Any]):
 
             # Handle the response based on the model output structure
             message = model_output.message
-            model_usage += model_output.usage
+            if model_output.usage is not None:
+                model_usage += model_output.usage
 
             current_messages.append(message)
             if isinstance(message, str):
                 result = message
             else:
-                result = message.content
+                result = message.text
 
             logger.debug(f'Turn:{turn_idx} Step:{n_steps} Result: {result}')
             current_responses.append(result)
@@ -115,7 +116,7 @@ def generate_turn(model: Model, row: dict[str, Any]):
 
             n_steps += 1
             if n_steps > MAXIMUM_STEP_LIMIT:
-                logger.error(f'INFERENCE_ERROR: Exceeded max inference steps ({MAXIMUM_STEP_LIMIT})')
+                logger.warning(f'INFERENCE_WARNING: Exceeded max inference steps ({MAXIMUM_STEP_LIMIT})')
                 break
 
         all_model_responses.append(current_responses)
@@ -145,9 +146,7 @@ def generate_turn_with_tools(model: Model, row: dict[str, Any]):
             new_tools = row['missing_functions'][str(turn_idx)]
             for new_tool in new_tools:
                 cur_tool = new_tool[0]
-                # change type to object
-                if cur_tool['parameters']['type'] != 'object':
-                    cur_tool['parameters']['type'] = 'object'
+                cur_tool['parameters']['type'] = 'object'
                 tools.append({
                     'type': 'function',
                     'function': cur_tool,
@@ -172,7 +171,8 @@ def generate_turn_with_tools(model: Model, row: dict[str, Any]):
 
             # Handle the response based on the model output structure
             message = model_output.message
-            model_usage += model_output.usage
+            if model_output.usage is not None:
+                model_usage += model_output.usage
 
             current_messages.append(message)
             if isinstance(message, str):
@@ -186,7 +186,7 @@ def generate_turn_with_tools(model: Model, row: dict[str, Any]):
                     logger.error(f'Error converting tool calls to function call strings: {e}')
                     tool_call_strs = None
             else:
-                model_responses = [message.content]
+                model_responses = [message.text]
                 tool_call_strs = None
 
             current_responses.extend(model_responses)
@@ -214,7 +214,7 @@ def generate_turn_with_tools(model: Model, row: dict[str, Any]):
 
             n_steps += 1
             if n_steps > MAXIMUM_STEP_LIMIT:
-                logger.error(f'INFERENCE_ERROR: Exceeded max inference steps ({MAXIMUM_STEP_LIMIT})')
+                logger.warning(f'INFERENCE_WARNING: Exceeded max inference steps ({MAXIMUM_STEP_LIMIT})')
                 break
 
         all_model_responses.append(current_responses)

evalscope/benchmarks/bfcl/v3/utils.py

@@ -0,0 +1,23 @@
+def convert_language(language: str) -> str:
+    """Convert language names from BFCL v3 to BFCL v4 naming conventions."""
+    from bfcl_eval.constants.enums import Language
+    mapping = {
+        'python': Language.PYTHON,
+        'java': Language.JAVA,
+        'javascript': Language.JAVASCRIPT,
+    }
+    return mapping[language.lower()]
+
+
+def convert_format_language(format_language: str) -> str:
+    """Convert format language names from BFCL v3 to BFCL v4 naming conventions."""
+    from bfcl_eval.constants.enums import ReturnFormat
+    mapping = {
+        'python': ReturnFormat.PYTHON,
+        'java': ReturnFormat.JAVA,
+        'javascript': ReturnFormat.JAVASCRIPT,
+        'json': ReturnFormat.JSON,
+        'verbose_xml': ReturnFormat.VERBOSE_XML,
+        'concise_xml': ReturnFormat.CONCISE_XML,
+    }
+    return mapping[format_language.lower()]

evalscope/benchmarks/bfcl/v4/bfcl_v4_adapter.py

@@ -0,0 +1,229 @@
+import json
+import os
+import traceback
+from copy import deepcopy
+from pathlib import Path
+from typing import Any, Dict, List
+
+from evalscope.api.benchmark import AgentAdapter, BenchmarkMeta
+from evalscope.api.dataset import Sample
+from evalscope.api.dataset.dataset import DatasetDict
+from evalscope.api.dataset.loader import DictDataLoader
+from evalscope.api.evaluator import TaskState
+from evalscope.api.messages.chat_message import ChatMessageUser
+from evalscope.api.metric import Score
+from evalscope.api.model import Model, ModelOutput
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.report import Report
+from evalscope.utils.function_utils import thread_safe
+from evalscope.utils.import_utils import check_import
+from evalscope.utils.logger import get_logger
+from .utils import (
+    ALL_SCORING_CATEGORIES,
+    compute_aggregate_subsets,
+    compute_entry_result,
+    load_bfcl_data,
+    process_test_entries,
+    run_prereq_inference,
+)
+
+logger = get_logger()
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='bfcl_v4',
+        pretty_name='BFCL-v4',
+        tags=[Tags.FUNCTION_CALLING, Tags.AGENT],
+        description='With function-calling being the building blocks of Agents, '
+        'the Berkeley Function-Calling Leaderboard (BFCL) V4 presents a holistic agentic '
+        'evaluation for LLMs. BFCL V4 Agentic includes web search, memory, and format sensitivity. '
+        'Together, the ability to web search, read and write from memory, and the ability to invoke '
+        'functions in different languages present the building blocks for the exciting and extremely '
+        'challenging avenues that power agentic LLMs today from deep-research, to agents for coding and law. '
+        'Need to run `pip install bfcl-eval==2025.10.27.1` before evaluating. '
+        '[Usage Example](https://evalscope.readthedocs.io/en/latest/third_party/bfcl_v4.html)',
+        dataset_id='https://github.com/ShishirPatil/gorilla/tree/main/berkeley-function-call-leaderboard',
+        subset_list=ALL_SCORING_CATEGORIES,
+        metric_list=['acc'],
+        eval_split='train',
+        extra_params={
+            'underscore_to_dot': True,
+            'is_fc_model': True,
+            'SERPAPI_API_KEY': None,
+        }
+    )
+)
+class BFCLV4Adapter(AgentAdapter):
+    """
+    BFCL adapter using the new data processing framework.
+    """
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+        check_import('bfcl_eval', package='bfcl-eval==2025.10.27.1', raise_error=True, feature_name=self.pretty_name)
+
+        self.add_overall_metric = False
+        self.add_aggregation_name = False
+
+        self.underscore_to_dot = self.extra_params.get('underscore_to_dot', True)
+        self.is_fc_model = self.extra_params.get('is_fc_model', True)
+        # Set SERPAPI_API_KEY in environment variables if provided
+        serpapi_api_key = self.extra_params.get('SERPAPI_API_KEY', None)
+        if serpapi_api_key:
+            os.environ['SERPAPI_API_KEY'] = serpapi_api_key
+        self.model_result_dir = Path(self._task_config.work_dir) if self._task_config else Path('./bfcl_model_results')
+        self.handler = None
+        self.prereq_entries = []
+        self.prereq_finished = False
+
+    def load(self):
+        """Load and process the BFCL dataset."""
+        from bfcl_eval.utils import parse_test_category_argument
+        datasets = {}
+        all_test_categories = parse_test_category_argument(self.subset_list)
+
+        test_entries_by_cat, ground_truth_by_cat = load_bfcl_data(all_test_categories)
+
+        for category in all_test_categories:
+            test_entries = test_entries_by_cat.get(category, [])
+            ground_truth_entries = ground_truth_by_cat.get(category, [])
+
+            if not test_entries:
+                continue
+
+            datasets[category] = self._create_dataset_for_category(category, test_entries, ground_truth_entries)
+
+        test_dataset = DatasetDict(datasets)
+        return test_dataset, None
+
+    def _create_dataset_for_category(
+        self, category: str, test_entries: List[Dict], ground_truth_entries: List[Dict]
+    ) -> DatasetDict:
+        """Create a dataset for a single category by merging test and ground truth data."""
+        processed_entries, prereq_entries = process_test_entries(
+            category=category,
+            test_entries=test_entries,
+            ground_truth_entries=ground_truth_entries,
+            model_result_dir=self.model_result_dir,
+        )
+        # collect prereq entries for later prereq inference
+        self.prereq_entries.extend(prereq_entries)
+
+        return DictDataLoader(
+            dict_list=processed_entries,
+            limit=self.limit,
+            repeats=self.repeats,
+            sample_fields=self.record_to_sample,
+            shuffle=self.shuffle,
+        ).load()
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        """Convert a data record to a Sample object."""
+        return Sample(
+            input=[ChatMessageUser(content=json.dumps(record['question']))],
+            target=json.dumps(record['ground_truth']),  # Will use the record for evaluation
+            metadata=record  # Store the full record for evaluation
+        )
+
+    @thread_safe
+    def _init_handler(self):
+        if self.handler is not None:
+            return  # Handler already initialized
+
+        from bfcl_eval.model_handler.api_inference.openai_completion import OpenAICompletionsHandler
+
+        # Set env variables for OpenAI API
+        os.environ['OPENAI_API_KEY'] = self._task_config.api_key
+        os.environ['OPENAI_BASE_URL'] = self._task_config.api_url
+
+        self.handler = OpenAICompletionsHandler(
+            model_name=self._task_config.model,
+            temperature=self._task_config.generation_config.temperature,
+            registry_name=self._task_config.model_id,
+            is_fc_model=self.is_fc_model,
+        )
+
+        self._prereq_inference()
+
+    def _prereq_inference(self):
+        if self.prereq_finished:
+            return
+        # MOVED: delegate prereq processing to utils
+        run_prereq_inference(
+            handler=self.handler,
+            prereq_entries=self.prereq_entries,
+            model_result_dir=self.model_result_dir,
+            batch_size=self._task_config.eval_batch_size,
+            logger=logger,
+        )
+        self.prereq_finished = True
+
+    def _on_inference(self, model: Model, sample: Sample) -> ModelOutput:
+        try:
+            self._init_handler()
+
+            result, _ = self.handler.inference(
+                deepcopy(sample.metadata), include_input_log=False, exclude_state_log=False
+            )
+
+            output = ModelOutput.from_content(
+                model=model.name,
+                content=json.dumps(result),
+            )
+        except Exception as e:
+            # This is usually the case when the model getting stuck on one particular test case.
+            # For example, timeout error or FC model returning invalid JSON response.
+            # Since temperature is already set to 0.001, retrying the same test case will not help.
+            # So we continue the generation process and record the error message as the model response
+            logger.error(f'Error during inference for sample ID {sample.metadata.get("id")}: {e}')
+            logger.error(traceback.format_exc())
+
+            output = ModelOutput.from_content(
+                model=model.name,
+                content=json.dumps({
+                    'error': str(e),
+                    'error_message': traceback.format_exc(),
+                }),
+            )
+        return output
+
+    def match_score(
+        self, original_prediction: str, filtered_prediction: str, reference: str, task_state: TaskState
+    ) -> Score:
+        self._init_handler()
+
+        score = Score(
+            extracted_prediction=filtered_prediction,
+            prediction=original_prediction,
+        )
+        model_result = json.loads(filtered_prediction)
+        prompt = task_state.metadata
+
+        entry_result = compute_entry_result(
+            handler=self.handler,
+            model_result=model_result,
+            prompt_entry=prompt,
+            underscore_to_dot=self.underscore_to_dot,
+        )
+
+        valid = 1 if entry_result['valid'] else 0
+        score.value = {'acc': valid}
+        score.metadata = {
+            'valid': bool(entry_result.get('valid')),
+            'error': str(entry_result.get('error')),
+            'error_message': str(entry_result.get('error_message')),
+            'error_type': str(entry_result.get('error_type')),
+        }
+        return score
+
+    def _on_generate_report_end(self, report: Report, output_dir, **kwargs):
+        """
+        Finalize the report generation process. Calculate the overall score.
+        """
+
+        # noqa: E501
+        # MOVED: delegate aggregation logic to utils
+        compute_aggregate_subsets(report)