evalscope 1.0.1__py3-none-any.whl → 1.1.0__py3-none-any.whl

evalscope/benchmarks/bbh/bbh_adapter.py

@@ -141,35 +141,61 @@ class BBHAdapter(DefaultDataAdapter):
     @classmethod
     def _extract_mc_answer(cls, ans: str) -> str:
         """
-        Extract the answer from the model output for Multiple choice task.
+        Extract normalized answer for BBH multiple-choice tasks.
+        Handles formats like:
+        - "answer is (A)"
+        - "The answer is A."
+        - Extra text after answer.
+        Always uses the *last* occurrence of "answer is".
         """
-        ans_line = ans.split('answer is ')
-        if len(ans_line) != 1:
-            ans = ans_line[1].strip()
-        match = re.search(r'\(([A-Z])\)*', ans)
+        ans = ans.strip()
+
+        parts = ans.split('So the answer is ')
+        if len(parts) > 1:
+            ans = parts[-1].strip()
+        ans = ans.split('\n')[0].strip()
+
+        # Remove trailing period
+        if ans.endswith('.'):
+            ans = ans[:-1].strip()
+
+        # Capture uppercase letter inside parentheses (A) (B) ...
+        match = re.search(r'\(([A-Z])\)', ans)
         if match:
             return match.group(1)
-        match = re.search(r'([A-Z])', ans)
+
+        # Capture single uppercase letter
+        match = re.search(r'\b([A-Z])\b', ans)
         if match:
             return match.group(1)
+
         return ans
 
     @classmethod
     def _extract_ff_answer(cls, ans: str):
         """
-        Extract the answer from the model output for Free-form task.
+        Extract the normalized answer for BBH free-form tasks.
+        Handles patterns like:
+        - "answer is XXX."
+        - "The answer is **valid**."
+        - Extra trailing dots / line breaks.
+        - Bold-marked answers (**xxx**).
+        Always uses the *last* occurrence of "answer is".
         """
-        pattern = r'answer is\s+(.*?)\.'
+        ans = ans.strip()
 
-        match = re.search(pattern, ans)
-        if match:
-            res = match.group(1)
-            return res
+        parts = ans.split('So the answer is ')
+        if len(parts) > 1:
+            ans = parts[-1].strip()
+        ans = ans.split('\n')[0].strip()
 
-        ans_line = ans.split('answer is ')
-        if len(ans_line) != 1:
-            ans = ans_line[1].strip()
-        ans = ans.split('\n')[0]
+        # Remove trailing period
         if ans.endswith('.'):
-            ans = ans[:-1]
+            ans = ans[:-1].strip()
+
+        # If answer is in bold (**xxx**), prefer the content inside
+        match = re.search(r'\*\*(.*?)\*\*', ans)
+        if match:
+            ans = match.group(1).strip()
+
         return ans

evalscope/benchmarks/bfcl/bfcl_adapter.py

@@ -1,7 +1,7 @@
 import json
 import re
 import traceback
-from typing import Any, Dict
+from typing import Any, Dict, List
 
 from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter
 from evalscope.api.dataset import Sample
@@ -11,6 +11,7 @@ from evalscope.api.metric import Score
 from evalscope.api.model import Model, ModelOutput
 from evalscope.api.registry import register_benchmark
 from evalscope.constants import Tags
+from evalscope.report import Category, Report, Subset, unweighted_average_from_subsets, weighted_average_from_subsets
 from evalscope.utils.import_utils import check_import
 from evalscope.utils.logger import get_logger
 
@@ -67,10 +68,12 @@ class BFCLAdapter(DefaultDataAdapter):
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
 
-        check_import('bfcl_eval', package='bfcl-eval==2025.6.16', raise_error=True)
+        check_import('bfcl_eval', package='bfcl-eval==2025.6.16', raise_error=True, feature_name=self.pretty_name)
 
         self.category_map = SUBJECT_MAPPING
         self.reformat_subset = True
+        self.add_overall_metric = False
+        self.add_aggregation_name = False
 
         self.underscore_to_dot = self.extra_params.get('underscore_to_dot', True)
         self.is_fc_model = self.extra_params.get('is_fc_model', True)
@@ -252,3 +255,104 @@ class BFCLAdapter(DefaultDataAdapter):
             score.metadata = {'error': traceback.format_exc()}
             score.main_score_name = 'acc'
         return score
+
+    def _on_generate_report_end(self, report: Report, output_dir, **kwargs):
+        """
+        Finalize the report generation process. Calculate the overall score.
+
+        Track the number of each category.
+        - step1: simple, java, javascript unweighted average as simple_ast
+        - step2.1: simple_ast, multiple, parallel, parallel_multiple unweighted average as ast_non_live
+        - step2.2: live_simple, live_multiple, live_parallel, live_parallel_multiple weighted average as ast_live
+        - step2.3: irrelevance as hallucination_non_live
+        - step2.4: live_irrelevance, live_relevance weighted average as hallucination_live
+        - step2.5: multi_turn_base as multi_turn_base
+        - step2.6: multi_turn_miss_func, multi_turn_miss_param, multi_turn_long_context weighted average as multi_turn_augmented
+        - step3.1: ast_non_live, hallucination_non_live unweighted average as non_live
+        - step3.2: ast_live, hallucination_live weighted average as live
+        - step3.3: multi_turn_base, multi_turn_augmented unweighted average as multi_turn
+        - step4: non_live, live, multi_turn unweighted average as overall
+        Args:
+            report (Report): The generated evaluation report.
+            output_dir (str): The directory to save the report.
+
+        Returns:
+            None
+        """  # noqa: E501
+        for metric in report.metrics:
+            # Collect all subsets in a dictionary for easy access
+            subset_dict: Dict[str, Subset] = {}
+            for category in metric.categories:
+                for subset in category.subsets:
+                    subset_dict[subset.name] = subset
+
+            # Step 1: Calculate simple_ast (simple, java, javascript unweighted average)
+            simple_subsets = ['simple', 'java', 'javascript']
+            simple_ast = unweighted_average_from_subsets(simple_subsets, subset_dict)
+            subset_dict['simple_ast'] = simple_ast
+
+            # Step 2.1: Calculate ast_non_live
+            # (simple_ast, multiple, parallel, parallel_multiple unweighted average)
+            ast_non_live_subsets = ['simple_ast', 'multiple', 'parallel', 'parallel_multiple']
+            ast_non_live = unweighted_average_from_subsets(ast_non_live_subsets, subset_dict)
+            subset_dict['ast_non_live'] = ast_non_live
+
+            # Step 2.2: Calculate ast_live
+            # (live_simple, live_multiple, live_parallel, live_parallel_multiple weighted average)
+            live_subsets = ['live_simple', 'live_multiple', 'live_parallel', 'live_parallel_multiple']
+            ast_live = weighted_average_from_subsets(live_subsets, subset_dict)
+            subset_dict['ast_live'] = ast_live
+
+            # Step 2.3: hallucination_non_live (irrelevance)
+            if 'irrelevance' in subset_dict:
+                subset_dict['hallucination_non_live'] = subset_dict['irrelevance']
+            else:
+                subset_dict['hallucination_non_live'] = Subset(name='hallucination_non_live', score=0, num=0)
+
+            # Step 2.4: Calculate hallucination_live (live_irrelevance, live_relevance weighted average)
+            hallucination_live_subsets = ['live_irrelevance', 'live_relevance']
+            hallucination_live = weighted_average_from_subsets(hallucination_live_subsets, subset_dict)
+            subset_dict['hallucination_live'] = hallucination_live
+
+            # Step 2.5: multi_turn_base
+            if 'multi_turn_base' not in subset_dict:
+                subset_dict['multi_turn_base'] = Subset(name='multi_turn_base', score=0, num=0)
+
+            # Step 2.6: Calculate multi_turn_augmented
+            # (multi_turn_miss_func, multi_turn_miss_param, multi_turn_long_context weighted average)
+            multi_turn_augmented_subsets = ['multi_turn_miss_func', 'multi_turn_miss_param', 'multi_turn_long_context']
+            multi_turn_augmented = weighted_average_from_subsets(multi_turn_augmented_subsets, subset_dict)
+            subset_dict['multi_turn_augmented'] = multi_turn_augmented
+
+            # Step 3.1: Calculate non_live (ast_non_live, hallucination_non_live unweighted average)
+            non_live_subsets = ['ast_non_live', 'hallucination_non_live']
+            non_live = unweighted_average_from_subsets(non_live_subsets, subset_dict)
+            subset_dict['non_live'] = non_live
+
+            # Step 3.2: Calculate live (ast_live, hallucination_live weighted average)
+            live_agg_subsets = ['ast_live', 'hallucination_live']
+            live = weighted_average_from_subsets(live_agg_subsets, subset_dict)
+            subset_dict['live'] = live
+
+            # Step 3.3: Calculate multi_turn (multi_turn_base, multi_turn_augmented unweighted average)
+            multi_turn_subsets = ['multi_turn_base', 'multi_turn_augmented']
+            multi_turn = unweighted_average_from_subsets(multi_turn_subsets, subset_dict)
+            subset_dict['multi_turn'] = multi_turn
+
+            # Step 4: Calculate overall (non_live, live, multi_turn unweighted average)
+            overall_subsets = ['non_live', 'live', 'multi_turn']
+            overall = unweighted_average_from_subsets(overall_subsets, subset_dict)
+            subset_dict['overall'] = overall
+
+            # Add computed scores to the category
+            computed_subset_names = ['non_live', 'live', 'multi_turn', 'overall']
+
+            # Add the computed scores as new subsets in the metric
+            dummy_subsets = []
+            for subset_name in computed_subset_names:
+                if subset_name in subset_dict:
+                    subset = subset_dict[subset_name]
+                    subset.name = subset_name.upper()
+                    dummy_subsets.append(subset)
+            dummy_category = Category(name='-', subsets=dummy_subsets)
+            metric.categories.append(dummy_category)

evalscope/benchmarks/bfcl/generation.py

@@ -72,7 +72,8 @@ def generate_turn(model: Model, row: dict[str, Any]):
 
             # Handle the response based on the model output structure
             message = model_output.message
-            model_usage += model_output.usage
+            if model_output.usage is not None:
+                model_usage += model_output.usage
 
             current_messages.append(message)
             if isinstance(message, str):
@@ -115,7 +116,7 @@ def generate_turn(model: Model, row: dict[str, Any]):
 
             n_steps += 1
             if n_steps > MAXIMUM_STEP_LIMIT:
-                logger.error(f'INFERENCE_ERROR: Exceeded max inference steps ({MAXIMUM_STEP_LIMIT})')
+                logger.warning(f'INFERENCE_WARNING: Exceeded max inference steps ({MAXIMUM_STEP_LIMIT})')
                 break
 
         all_model_responses.append(current_responses)
@@ -145,9 +146,7 @@ def generate_turn_with_tools(model: Model, row: dict[str, Any]):
             new_tools = row['missing_functions'][str(turn_idx)]
             for new_tool in new_tools:
                 cur_tool = new_tool[0]
-                # change type to object
-                if cur_tool['parameters']['type'] != 'object':
-                    cur_tool['parameters']['type'] = 'object'
+                cur_tool['parameters']['type'] = 'object'
                 tools.append({
                     'type': 'function',
                     'function': cur_tool,
@@ -172,7 +171,8 @@ def generate_turn_with_tools(model: Model, row: dict[str, Any]):
 
             # Handle the response based on the model output structure
             message = model_output.message
-            model_usage += model_output.usage
+            if model_output.usage is not None:
+                model_usage += model_output.usage
 
             current_messages.append(message)
             if isinstance(message, str):
@@ -214,7 +214,7 @@ def generate_turn_with_tools(model: Model, row: dict[str, Any]):
 
             n_steps += 1
             if n_steps > MAXIMUM_STEP_LIMIT:
-                logger.error(f'INFERENCE_ERROR: Exceeded max inference steps ({MAXIMUM_STEP_LIMIT})')
+                logger.warning(f'INFERENCE_WARNING: Exceeded max inference steps ({MAXIMUM_STEP_LIMIT})')
                 break
 
         all_model_responses.append(current_responses)

evalscope/benchmarks/blink/blink_adapter.py

@@ -0,0 +1,61 @@
+import re
+from typing import Any, Dict, List
+
+from evalscope.api.benchmark import BenchmarkMeta, MultiChoiceAdapter, VisionLanguageAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.messages import ChatMessageUser, Content, ContentImage, ContentText
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.io_utils import bytes_to_base64
+from evalscope.utils.logger import get_logger
+from evalscope.utils.multi_choices import format_letter_choices
+
+logger = get_logger()
+
+MULT_CHOICE_PROMPT = r"""
+Answer the following multiple choice question. The last line of your response should be of the following format:
+'ANSWER: $LETTER' (without quotes) where LETTER is one of {letters}.
+
+{question}
+""".strip()
+
+SUBSET_LIST = [
+    'Art_Style', 'Counting', 'Forensic_Detection', 'Functional_Correspondence', 'IQ_Test', 'Jigsaw',
+    'Multi-view_Reasoning', 'Object_Localization', 'Relative_Depth', 'Relative_Reflectance', 'Semantic_Correspondence',
+    'Spatial_Relation', 'Visual_Correspondence', 'Visual_Similarity'
+]
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='blink',
+        pretty_name='BLINK',
+        tags=[Tags.MULTI_MODAL, Tags.KNOWLEDGE, Tags.MULTIPLE_CHOICE],
+        description=
+        'BLINK is a benchmark designed to evaluate the core visual perception abilities of multimodal large language models (MLLMs). It transforms 14 classic computer vision tasks into 3,807 multiple-choice questions, accompanied by single or multiple images and visual prompts.',  # noqa: E501
+        dataset_id='evalscope/BLINK',
+        subset_list=SUBSET_LIST,
+        metric_list=['acc'],
+        eval_split='val',
+        prompt_template=MULT_CHOICE_PROMPT,
+    )
+)
+class BLINKAdapter(VisionLanguageAdapter, MultiChoiceAdapter):
+    MAX_IMAGES: int = 4
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        choices = record.get('choices')
+        input_text = MULT_CHOICE_PROMPT.format(question=record['prompt'], letters=format_letter_choices(choices))
+        content_list: List[Content] = [ContentText(text=input_text)]
+
+        for i in range(1, self.MAX_IMAGES + 1):
+            image = record.get(f'image_{i}')
+            if image:
+                image_base64 = bytes_to_base64(image['bytes'], format='jpeg', add_header=True)
+                content_list.append(ContentImage(image=image_base64))
+
+        label_answer = record['answer'].strip('(').strip(')')
+        return Sample(input=[ChatMessageUser(content=content_list)], choices=choices, target=label_answer)

evalscope/benchmarks/chartqa/chartqa_adapter.py

@@ -0,0 +1,80 @@
+import re
+from typing import Any, Dict, List
+
+from evalscope.api.benchmark import BenchmarkMeta, VisionLanguageAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.evaluator import TaskState
+from evalscope.api.messages import ChatMessageUser, Content, ContentImage, ContentText
+from evalscope.api.metric.scorer import Score
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.io_utils import bytes_to_base64
+from evalscope.utils.logger import get_logger
+
+# flake8: noqa
+
+logger = get_logger()
+
+OPEN_PROMPT = """
+{question}
+
+The last line of your response should be of the form "ANSWER: $ANSWER" (without quotes) where $ANSWER is the a single word answer to the problem.
+"""
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='chartqa',
+        pretty_name='ChartQA',
+        tags=[Tags.MULTI_MODAL, Tags.KNOWLEDGE, Tags.QA],
+        description=
+        'ChartQA is a benchmark designed to evaluate question-answering capabilities about charts (e.g., bar charts, line graphs, pie charts), focusing on both visual and logical reasoning.',  # noqa: E501
+        dataset_id='lmms-lab/ChartQA',
+        subset_list=['human_test', 'augmented_test'],
+        metric_list=['relaxed_acc'],
+        eval_split='test',
+        prompt_template=OPEN_PROMPT,
+    )
+)
+class ChartQAAdapter(VisionLanguageAdapter):
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        self.add_aggregation_name = False
+        self.reformat_subset = True
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        question = record['question']
+        image_data = record['image']
+        image_base64 = bytes_to_base64(image_data['bytes'], format='png', add_header=True)
+
+        content_list: List[Content] = [
+            ContentText(text=OPEN_PROMPT.format(question=question)),
+            ContentImage(image=image_base64)
+        ]
+
+        return Sample(
+            input=[ChatMessageUser(content=content_list)],
+            target=record['answer'],
+            subset_key=record['type'],  # 'human_test' or 'augmented_split'
+        )
+
+    def extract_answer(self, prediction: str, task_state: TaskState) -> str:
+        pattern = r'ANSWER:\s*(.*)'
+        match = re.search(pattern, prediction)
+        if match:
+            return match.group(1).strip()
+        return ''
+
+    def match_score(self, original_prediction, filtered_prediction, reference, task_state) -> Score:
+        from .utils import relaxed_correctness
+
+        score = relaxed_correctness(filtered_prediction, reference)
+        score = 1.0 if score else 0.0
+
+        return Score(
+            value={'relaxed_acc': score},
+            prediction=original_prediction,
+            extracted_prediction=filtered_prediction,
+        )

evalscope/benchmarks/chartqa/utils.py

@@ -0,0 +1,38 @@
+def relaxed_correctness(prediction: str, target: str, max_relative_change: float = 0.05) -> bool:
+    """Calculates relaxed correctness.
+
+    The correctness tolerates certain error ratio defined by max_relative_change.
+    See https://arxiv.org/pdf/2203.10244.pdf, end of section 5.1:
+    “Following Methani et al. (2020), we use a relaxed accuracy measure for the
+    numeric answers to allow a minor inaccuracy that may result from the automatic
+    data extraction process. We consider an answer to be correct if it is within
+    5% of the gold answer. For non-numeric answers, we still need an exact match
+    to consider an answer to be correct.”
+
+    This funcion is taken from https://github.com/QwenLM/Qwen-VL/blob/34b4c0ee7b07726371b960911f249fe61b362ca3/eval_mm/evaluate_vqa.py#L113
+    Args:
+      target: List of target string.
+      prediction: List of predicted string.
+      max_relative_change: Maximum relative change.
+
+    Returns:
+      Whether the prediction was correct given the specified tolerance.
+    """  # noqa: E501
+
+    def _to_float(text: str):
+        try:
+            if text.endswith('%'):
+                # Convert percentages to floats.
+                return float(text.rstrip('%')) / 100.0
+            else:
+                return float(text)
+        except ValueError:
+            return None
+
+    prediction_float = _to_float(prediction)
+    target_float = _to_float(target)
+    if prediction_float is not None and target_float:
+        relative_change = abs(prediction_float - target_float) / abs(target_float)
+        return relative_change <= max_relative_change
+    else:
+        return prediction.lower() == target.lower()

evalscope/benchmarks/docvqa/docvqa_adapter.py

@@ -0,0 +1,67 @@
+import json
+from typing import Any, Dict, List
+
+from evalscope.api.benchmark import BenchmarkMeta, VisionLanguageAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.evaluator.state import TaskState
+from evalscope.api.messages import ChatMessageUser, Content, ContentImage, ContentText
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.io_utils import bytes_to_base64
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+PROMPT = """Answer the question according to the image using a single word or phrase.
+{question}
+The last line of your response should be of the form "ANSWER: $ANSWER" (without quotes) where $ANSWER is the answer to the question."""  # noqa: E501
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='docvqa',
+        pretty_name='DocVQA',
+        tags=[Tags.MULTI_MODAL, Tags.KNOWLEDGE, Tags.QA],
+        description=
+        'DocVQA (Document Visual Question Answering) is a benchmark designed to evaluate AI systems on their ability to answer questions based on the content of document images, such as scanned pages, forms, or invoices. Unlike general visual question answering, it requires understanding not just the text extracted by OCR, but also the complex layout, structure, and visual elements of a document.',  # noqa: E501
+        dataset_id='lmms-lab/DocVQA',
+        subset_list=['DocVQA'],
+        metric_list=['anls'],
+        eval_split='validation',
+        prompt_template=PROMPT,
+    )
+)
+class DocVQAAdapter(VisionLanguageAdapter):
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.add_aggregation_name = False
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+
+        input_text = PROMPT.format(question=record['question'])
+        content_list: List[Content] = [ContentText(text=input_text)]
+        image = record.get('image')
+        if image:
+            image_base64 = bytes_to_base64(image['bytes'], format='png', add_header=True)
+            content_list.append(ContentImage(image=image_base64))
+        return Sample(
+            input=[ChatMessageUser(content=content_list)],
+            target=json.dumps(record.get('answers')),  # answers is a list
+            metadata={
+                'questionId': record.get('questionId'),
+                'question_types': record.get('question_types'),
+                'docId': record.get('docId'),
+                'ucsf_document_id': record.get('ucsf_document_id'),
+                'ucsf_document_page_no': record.get('ucsf_document_page_no'),
+            }
+        )
+
+    def extract_answer(self, prediction: str, task_state: TaskState) -> str:
+        import re
+
+        pattern = r'ANSWER:\s*(.*)'
+        match = re.search(pattern, prediction)
+        if match:
+            return match.group(1).strip()
+        return prediction.strip()

evalscope/benchmarks/drop/drop_adapter.py

@@ -54,7 +54,7 @@ class DROPAdapter(DefaultDataAdapter):
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
 
-        if self.few_shot_num != 0:
+        if self.few_shot_num != 0 and self.few_shot_num != 3:
             self.few_shot_num = 3
             logger.info(f'Few shot num is set to {self.few_shot_num} for DROP dataset by system.')
         else:

evalscope/benchmarks/general_arena/utils.py

@@ -34,7 +34,8 @@ def process_review_item(review_result: ReviewResult) -> list:
         'Index': str(review_result.index),
         'Input': review_result.input,
         'Question': review_result.input,  # Use input as question
-        'Generated': prediction if prediction != extracted_prediction else extracted_prediction,
+        'Generated':
+        prediction if prediction != extracted_prediction else extracted_prediction or '',  # Ensure no None value
         'Gold': target,
         'Pred': extracted_prediction,
         'Score': sample_score.score.model_dump(exclude_none=True),