evalscope 1.0.2__py3-none-any.whl → 1.1.1__py3-none-any.whl

evalscope/benchmarks/aa_lcr/aa_lcr_adapter.py

@@ -0,0 +1,205 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+# flake8: noqa: E501
+import re
+import urllib.request
+import zipfile
+from pathlib import Path
+from typing import Any, Dict
+
+from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.evaluator import TaskState
+from evalscope.api.messages import ChatMessageUser
+from evalscope.api.metric import Score
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import DEFAULT_EVALSCOPE_CACHE_DIR, Tags
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+# Default judge prompt template
+JUDGE_PROMPT = """Assess whether the following CANDIDATE ANSWER is CORRECT or INCORRECT. For the CANDIDATE ANSWER to be correct, it must be consistent with the OFFICIAL ANSWER.
+
+The question, for reference only: {question}
+The OFFICIAL ANSWER: {correct_answer}
+CANDIDATE ANSWER TO ASSESS: {response}
+
+Reply only with CORRECT or INCORRECT."""
+
+PROMPT_TEMPLATE = """
+BEGIN INPUT DOCUMENTS
+
+{documents_text}
+
+END INPUT DOCUMENTS
+
+Answer the following question using the input documents provided above.
+
+START QUESTION
+
+{question}
+
+END QUESTION
+"""
+
+# New constants for auto-download
+DOWNLOAD_URL: str = (
+    'https://modelscope.cn/datasets/evalscope/AA-LCR/resolve/master/extracted_text/AA-LCR_extracted-text.zip'
+)
+DEFAULT_CACHE_SUBDIR: str = 'aa_lcr'
+DEFAULT_ZIP_NAME: str = 'AA-LCR_extracted-text.zip'
+DEFAULT_EXTRACTED_DIR_NAME: str = 'lcr'
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='aa_lcr',
+        pretty_name='AA-LCR',
+        tags=[Tags.KNOWLEDGE, Tags.REASONING, Tags.LONG_CONTEXT],
+        description='AA-LCR (Artificial Analysis Long Context Retrieval) is a benchmark for evaluating long-context '
+        'retrieval and reasoning capabilities of language models across multiple documents.',  # noqa: E501
+        dataset_id='evalscope/AA-LCR',
+        metric_list=['acc'],
+        few_shot_num=0,
+        train_split=None,
+        eval_split='test',
+        prompt_template=PROMPT_TEMPLATE,
+        extra_params={'text_dir': None}
+    )
+)
+class AALCRAdapter(DefaultDataAdapter):
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        self._use_llm_judge = True
+
+        # Get extra parameters
+        self.text_dir = self.extra_params.get('text_dir')
+
+    def load(self):
+        # Auto download and extract when text_dir is not provided
+        if not self.text_dir:
+            self.text_dir = self._ensure_text_dir_downloaded()
+        elif not Path(self.text_dir).exists():
+            raise ValueError(
+                'AA-LCR text_dir does not exist: '
+                f'{self.text_dir}. Please provide a valid directory or omit text_dir to auto-download.'
+            )
+
+        self.text_dir = Path(self.text_dir)
+        return super().load()
+
+    def _ensure_text_dir_downloaded(self) -> Path:
+        """Ensure AA-LCR extracted texts are available locally; download and extract if missing."""
+        cache_root = Path(DEFAULT_EVALSCOPE_CACHE_DIR) / DEFAULT_CACHE_SUBDIR
+        extracted_dir = cache_root / DEFAULT_EXTRACTED_DIR_NAME
+
+        if extracted_dir.exists():
+            logger.info(f'AA-LCR documents found: {extracted_dir}')
+            return extracted_dir
+
+        cache_root.mkdir(parents=True, exist_ok=True)
+        zip_path = cache_root / DEFAULT_ZIP_NAME
+
+        try:
+            logger.info(f'Downloading AA-LCR documents from {DOWNLOAD_URL} to {zip_path}...')
+            urllib.request.urlretrieve(DOWNLOAD_URL, zip_path)
+
+            logger.info(f'Extracting {zip_path} to {cache_root}...')
+            with zipfile.ZipFile(zip_path, 'r') as zf:
+                zf.extractall(cache_root)
+
+            if not extracted_dir.exists():
+                raise ValueError(f'Extraction succeeded but target directory not found: {extracted_dir}')
+
+            logger.info(f'AA-LCR documents ready at {extracted_dir}')
+            return extracted_dir
+        except Exception as e:
+            raise ValueError(
+                f'Failed to download or extract AA-LCR documents: {e}. '
+                'You can also manually download and set extra_params["text_dir"].'
+            ) from e
+        finally:
+            # Best-effort cleanup of the zip file
+            try:
+                if zip_path.exists():
+                    zip_path.unlink()
+            except Exception:
+                pass
+
+    def _get_context(self, record: Dict[str, Any]) -> str:
+        doc_folder = self.text_dir / record['document_category'] / record['document_set_id']
+
+        # Check if the document folder exists
+        if not doc_folder.exists() or not doc_folder.is_dir():
+            logger.warning(f'Document folder not found: {doc_folder}. Returning empty context.')
+            return ''
+
+        doc_blocks = []
+        try:
+            for file_path in doc_folder.iterdir():
+                if file_path.is_file():
+                    try:
+                        content = file_path.read_text(encoding='utf-8').strip()
+                        if content:
+                            doc_blocks.append(content)
+                    except (IOError, UnicodeDecodeError) as e:
+                        logger.warning(f'Could not read file {file_path}, skipping: {e}')
+        except OSError as e:
+            logger.warning(f'Could not access document folder {doc_folder}: {e}')
+            return f"ERROR: Could not read documents for {record['document_category']}/{record['document_set_id']}"
+
+        documents_text = '\n\n'.join(
+            f'BEGIN DOCUMENT {i + 1}:\n{doc}\nEND DOCUMENT {i + 1}' for i, doc in enumerate(doc_blocks)
+        )
+        return documents_text
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        """Convert a record to a Sample with long-context prompt."""
+        context = self._get_context(record)
+        prompt = self.prompt_template.format(documents_text=context, question=record['question'])
+
+        return Sample(
+            input=[ChatMessageUser(content=prompt)],
+            target=record['answer'],
+            metadata={
+                'question': record['question'],
+                'data_source_urls': record['data_source_urls'],
+                'input_tokens': record.get('input_tokens', 0),
+            }
+        )
+
+    def llm_match_score(
+        self,
+        original_prediction: str,
+        filtered_prediction: str,
+        reference: str,
+        task_state: TaskState,
+    ) -> Score:
+        score = Score(
+            extracted_prediction=filtered_prediction,
+            prediction=original_prediction,
+        )
+
+        judge_prompt = JUDGE_PROMPT.format(
+            question=task_state.metadata['question'], correct_answer=reference, response=filtered_prediction
+        )
+
+        # Request judge and obtain score
+        judge_response = self.llm_judge.judge(prompt=judge_prompt)
+
+        # Parse judge response to get accuracy score
+        # Use word boundaries to avoid matching "CORRECT" within "INCORRECT"
+        is_correct = bool(re.search(r'\bCORRECT\b', judge_response, re.IGNORECASE))
+        score.value = {
+            'acc': 1.0 if is_correct else 0.0,
+        }
+        score.explanation = f'LLM judge: {judge_response}'
+        score.metadata = {
+            'source': 'llm_judge',
+            'judge_strategy': self.judge_strategy,
+            'model': self.llm_judge.model_id,
+        }
+        score.main_score_name = 'acc'
+        return score

evalscope/benchmarks/ai2d/ai2d_adapter.py

@@ -22,7 +22,8 @@ MULT_CHOICE_PROMPT = MultipleChoiceTemplate.SINGLE_ANSWER_COT
         name='ai2d',
         pretty_name='AI2D',
         tags=[Tags.MULTI_MODAL, Tags.KNOWLEDGE, Tags.QA],
-        description='A Diagram Is Worth A Dozen Images',
+        description=
+        'AI2D is a benchmark dataset for researching the understanding of diagrams by AI. It contains over 5,000 diverse diagrams from science textbooks (e.g., the water cycle, food webs). Each diagram is accompanied by multiple-choice questions that test an AI\'s ability to interpret visual elements, text labels, and their relationships. The benchmark is challenging because it requires jointly understanding the layout, symbols, and text to answer questions correctly.',  # noqa: E501
         dataset_id='lmms-lab/ai2d',
         subset_list=SUBSET_LIST,
         metric_list=['acc'],
@@ -37,7 +38,7 @@ class Ai2dAdapter(VisionLanguageAdapter):
 
     def record_to_sample(self, record: Dict[str, Any]) -> Sample:
         answers_list: list[str] = record['options']
-        input_text = prompt(question=record['question'], choices=answers_list, template=MULT_CHOICE_PROMPT)
+        input_text = prompt(question=record['question'], choices=answers_list, template=self.prompt_template)
         content_list: list[Content] = [ContentText(text=input_text)]
         image = record.get('image')
         if image:

evalscope/benchmarks/bfcl/bfcl_adapter.py

@@ -8,11 +8,10 @@ from evalscope.api.dataset import Sample
 from evalscope.api.evaluator import TaskState
 from evalscope.api.messages.chat_message import ChatMessageUser
 from evalscope.api.metric import Score
-from evalscope.api.metric.scorer import AggScore
 from evalscope.api.model import Model, ModelOutput
 from evalscope.api.registry import register_benchmark
 from evalscope.constants import Tags
-from evalscope.report import Category, Report, Subset
+from evalscope.report import Category, Report, Subset, unweighted_average_from_subsets, weighted_average_from_subsets
 from evalscope.utils.import_utils import check_import
 from evalscope.utils.logger import get_logger
 
@@ -50,7 +49,7 @@ SUBJECT_MAPPING = {
         'functions. Unlike previous evaluations, '
         'BFCL accounts for various forms of function calls, diverse scenarios, and executability. '
         'Need to run `pip install bfcl-eval==2025.6.16` before evaluating. '
-        '[Usage Example](https://evalscope.readthedocs.io/zh-cn/latest/third_party/bfcl_v3.html)',
+        '[Usage Example](https://evalscope.readthedocs.io/en/latest/third_party/bfcl_v3.html)',
         dataset_id='AI-ModelScope/bfcl_v3',
         subset_list=list(SUBJECT_MAPPING.keys()),
         metric_list=['acc'],
@@ -79,40 +78,6 @@ class BFCLAdapter(DefaultDataAdapter):
         self.underscore_to_dot = self.extra_params.get('underscore_to_dot', True)
         self.is_fc_model = self.extra_params.get('is_fc_model', True)
 
-    def _weighted_average_from_subsets(self, subset_names: List[str], subset_dict: Dict[str, Subset]) -> Subset:
-        """Calculate weighted average for given subsets.
-
-        Returns:
-            Subset: A new Subset object with weighted average score
-        """
-        total_score = 0
-        total_count = 0
-        for name in subset_names:
-            if name in subset_dict:
-                subset = subset_dict[name]
-                total_score += subset.score * subset.num
-                total_count += subset.num
-
-        weighted_avg = total_score / total_count if total_count > 0 else 0
-        return Subset(name='', score=weighted_avg, num=total_count)
-
-    def _unweighted_average_from_subsets(self, subset_names: List[str], subset_dict: Dict[str, Subset]) -> Subset:
-        """Calculate unweighted average for given subsets.
-
-        Returns:
-            Subset: A new Subset object with unweighted average score
-        """
-        scores = []
-        total_count = 0
-        for name in subset_names:
-            if name in subset_dict:
-                subset = subset_dict[name]
-                scores.append(subset.score)
-                total_count += subset.num
-
-        unweighted_avg = sum(scores) / len(scores) if scores else 0
-        return Subset(name='', score=unweighted_avg, num=total_count)
-
     def preprocess_row(self, row: dict):
         """
         Inplace preprocess the row to ensure it has the correct format for BFCL evaluation.
@@ -323,19 +288,19 @@ class BFCLAdapter(DefaultDataAdapter):
 
             # Step 1: Calculate simple_ast (simple, java, javascript unweighted average)
             simple_subsets = ['simple', 'java', 'javascript']
-            simple_ast = self._unweighted_average_from_subsets(simple_subsets, subset_dict)
+            simple_ast = unweighted_average_from_subsets(simple_subsets, subset_dict)
             subset_dict['simple_ast'] = simple_ast
 
             # Step 2.1: Calculate ast_non_live
             # (simple_ast, multiple, parallel, parallel_multiple unweighted average)
             ast_non_live_subsets = ['simple_ast', 'multiple', 'parallel', 'parallel_multiple']
-            ast_non_live = self._unweighted_average_from_subsets(ast_non_live_subsets, subset_dict)
+            ast_non_live = unweighted_average_from_subsets(ast_non_live_subsets, subset_dict)
             subset_dict['ast_non_live'] = ast_non_live
 
             # Step 2.2: Calculate ast_live
             # (live_simple, live_multiple, live_parallel, live_parallel_multiple weighted average)
             live_subsets = ['live_simple', 'live_multiple', 'live_parallel', 'live_parallel_multiple']
-            ast_live = self._weighted_average_from_subsets(live_subsets, subset_dict)
+            ast_live = weighted_average_from_subsets(live_subsets, subset_dict)
             subset_dict['ast_live'] = ast_live
 
             # Step 2.3: hallucination_non_live (irrelevance)
@@ -346,7 +311,7 @@ class BFCLAdapter(DefaultDataAdapter):
 
             # Step 2.4: Calculate hallucination_live (live_irrelevance, live_relevance weighted average)
             hallucination_live_subsets = ['live_irrelevance', 'live_relevance']
-            hallucination_live = self._weighted_average_from_subsets(hallucination_live_subsets, subset_dict)
+            hallucination_live = weighted_average_from_subsets(hallucination_live_subsets, subset_dict)
             subset_dict['hallucination_live'] = hallucination_live
 
             # Step 2.5: multi_turn_base
@@ -356,27 +321,27 @@ class BFCLAdapter(DefaultDataAdapter):
             # Step 2.6: Calculate multi_turn_augmented
             # (multi_turn_miss_func, multi_turn_miss_param, multi_turn_long_context weighted average)
             multi_turn_augmented_subsets = ['multi_turn_miss_func', 'multi_turn_miss_param', 'multi_turn_long_context']
-            multi_turn_augmented = self._weighted_average_from_subsets(multi_turn_augmented_subsets, subset_dict)
+            multi_turn_augmented = weighted_average_from_subsets(multi_turn_augmented_subsets, subset_dict)
             subset_dict['multi_turn_augmented'] = multi_turn_augmented
 
             # Step 3.1: Calculate non_live (ast_non_live, hallucination_non_live unweighted average)
             non_live_subsets = ['ast_non_live', 'hallucination_non_live']
-            non_live = self._unweighted_average_from_subsets(non_live_subsets, subset_dict)
+            non_live = unweighted_average_from_subsets(non_live_subsets, subset_dict)
             subset_dict['non_live'] = non_live
 
             # Step 3.2: Calculate live (ast_live, hallucination_live weighted average)
             live_agg_subsets = ['ast_live', 'hallucination_live']
-            live = self._weighted_average_from_subsets(live_agg_subsets, subset_dict)
+            live = weighted_average_from_subsets(live_agg_subsets, subset_dict)
             subset_dict['live'] = live
 
             # Step 3.3: Calculate multi_turn (multi_turn_base, multi_turn_augmented unweighted average)
             multi_turn_subsets = ['multi_turn_base', 'multi_turn_augmented']
-            multi_turn = self._unweighted_average_from_subsets(multi_turn_subsets, subset_dict)
+            multi_turn = unweighted_average_from_subsets(multi_turn_subsets, subset_dict)
             subset_dict['multi_turn'] = multi_turn
 
             # Step 4: Calculate overall (non_live, live, multi_turn unweighted average)
             overall_subsets = ['non_live', 'live', 'multi_turn']
-            overall = self._unweighted_average_from_subsets(overall_subsets, subset_dict)
+            overall = unweighted_average_from_subsets(overall_subsets, subset_dict)
             subset_dict['overall'] = overall
 
             # Add computed scores to the category

evalscope/benchmarks/blink/blink_adapter.py

@@ -0,0 +1,61 @@
+import re
+from typing import Any, Dict, List
+
+from evalscope.api.benchmark import BenchmarkMeta, MultiChoiceAdapter, VisionLanguageAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.messages import ChatMessageUser, Content, ContentImage, ContentText
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.io_utils import bytes_to_base64
+from evalscope.utils.logger import get_logger
+from evalscope.utils.multi_choices import format_letter_choices
+
+logger = get_logger()
+
+MULT_CHOICE_PROMPT = r"""
+Answer the following multiple choice question. The last line of your response should be of the following format:
+'ANSWER: $LETTER' (without quotes) where LETTER is one of {letters}.
+
+{question}
+""".strip()
+
+SUBSET_LIST = [
+    'Art_Style', 'Counting', 'Forensic_Detection', 'Functional_Correspondence', 'IQ_Test', 'Jigsaw',
+    'Multi-view_Reasoning', 'Object_Localization', 'Relative_Depth', 'Relative_Reflectance', 'Semantic_Correspondence',
+    'Spatial_Relation', 'Visual_Correspondence', 'Visual_Similarity'
+]
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='blink',
+        pretty_name='BLINK',
+        tags=[Tags.MULTI_MODAL, Tags.KNOWLEDGE, Tags.MULTIPLE_CHOICE],
+        description=
+        'BLINK is a benchmark designed to evaluate the core visual perception abilities of multimodal large language models (MLLMs). It transforms 14 classic computer vision tasks into 3,807 multiple-choice questions, accompanied by single or multiple images and visual prompts.',  # noqa: E501
+        dataset_id='evalscope/BLINK',
+        subset_list=SUBSET_LIST,
+        metric_list=['acc'],
+        eval_split='val',
+        prompt_template=MULT_CHOICE_PROMPT,
+    )
+)
+class BLINKAdapter(VisionLanguageAdapter, MultiChoiceAdapter):
+    MAX_IMAGES: int = 4
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        choices = record.get('choices')
+        input_text = MULT_CHOICE_PROMPT.format(question=record['prompt'], letters=format_letter_choices(choices))
+        content_list: List[Content] = [ContentText(text=input_text)]
+
+        for i in range(1, self.MAX_IMAGES + 1):
+            image = record.get(f'image_{i}')
+            if image:
+                image_base64 = bytes_to_base64(image['bytes'], format='jpeg', add_header=True)
+                content_list.append(ContentImage(image=image_base64))
+
+        label_answer = record['answer'].strip('(').strip(')')
+        return Sample(input=[ChatMessageUser(content=content_list)], choices=choices, target=label_answer)

evalscope/benchmarks/chartqa/chartqa_adapter.py

@@ -0,0 +1,80 @@
+import re
+from typing import Any, Dict, List
+
+from evalscope.api.benchmark import BenchmarkMeta, VisionLanguageAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.evaluator import TaskState
+from evalscope.api.messages import ChatMessageUser, Content, ContentImage, ContentText
+from evalscope.api.metric.scorer import Score
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.io_utils import bytes_to_base64
+from evalscope.utils.logger import get_logger
+
+# flake8: noqa
+
+logger = get_logger()
+
+OPEN_PROMPT = """
+{question}
+
+The last line of your response should be of the form "ANSWER: $ANSWER" (without quotes) where $ANSWER is the a single word answer to the problem.
+"""
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='chartqa',
+        pretty_name='ChartQA',
+        tags=[Tags.MULTI_MODAL, Tags.KNOWLEDGE, Tags.QA],
+        description=
+        'ChartQA is a benchmark designed to evaluate question-answering capabilities about charts (e.g., bar charts, line graphs, pie charts), focusing on both visual and logical reasoning.',  # noqa: E501
+        dataset_id='lmms-lab/ChartQA',
+        subset_list=['human_test', 'augmented_test'],
+        metric_list=['relaxed_acc'],
+        eval_split='test',
+        prompt_template=OPEN_PROMPT,
+    )
+)
+class ChartQAAdapter(VisionLanguageAdapter):
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        self.add_aggregation_name = False
+        self.reformat_subset = True
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        question = record['question']
+        image_data = record['image']
+        image_base64 = bytes_to_base64(image_data['bytes'], format='png', add_header=True)
+
+        content_list: List[Content] = [
+            ContentText(text=OPEN_PROMPT.format(question=question)),
+            ContentImage(image=image_base64)
+        ]
+
+        return Sample(
+            input=[ChatMessageUser(content=content_list)],
+            target=record['answer'],
+            subset_key=record['type'],  # 'human_test' or 'augmented_split'
+        )
+
+    def extract_answer(self, prediction: str, task_state: TaskState) -> str:
+        pattern = r'ANSWER:\s*(.*)'
+        match = re.search(pattern, prediction)
+        if match:
+            return match.group(1).strip()
+        return ''
+
+    def match_score(self, original_prediction, filtered_prediction, reference, task_state) -> Score:
+        from .utils import relaxed_correctness
+
+        score = relaxed_correctness(filtered_prediction, reference)
+        score = 1.0 if score else 0.0
+
+        return Score(
+            value={'relaxed_acc': score},
+            prediction=original_prediction,
+            extracted_prediction=filtered_prediction,
+        )

evalscope/benchmarks/chartqa/utils.py

@@ -0,0 +1,38 @@
+def relaxed_correctness(prediction: str, target: str, max_relative_change: float = 0.05) -> bool:
+    """Calculates relaxed correctness.
+
+    The correctness tolerates certain error ratio defined by max_relative_change.
+    See https://arxiv.org/pdf/2203.10244.pdf, end of section 5.1:
+    “Following Methani et al. (2020), we use a relaxed accuracy measure for the
+    numeric answers to allow a minor inaccuracy that may result from the automatic
+    data extraction process. We consider an answer to be correct if it is within
+    5% of the gold answer. For non-numeric answers, we still need an exact match
+    to consider an answer to be correct.”
+
+    This funcion is taken from https://github.com/QwenLM/Qwen-VL/blob/34b4c0ee7b07726371b960911f249fe61b362ca3/eval_mm/evaluate_vqa.py#L113
+    Args:
+      target: List of target string.
+      prediction: List of predicted string.
+      max_relative_change: Maximum relative change.
+
+    Returns:
+      Whether the prediction was correct given the specified tolerance.
+    """  # noqa: E501
+
+    def _to_float(text: str):
+        try:
+            if text.endswith('%'):
+                # Convert percentages to floats.
+                return float(text.rstrip('%')) / 100.0
+            else:
+                return float(text)
+        except ValueError:
+            return None
+
+    prediction_float = _to_float(prediction)
+    target_float = _to_float(target)
+    if prediction_float is not None and target_float:
+        relative_change = abs(prediction_float - target_float) / abs(target_float)
+        return relative_change <= max_relative_change
+    else:
+        return prediction.lower() == target.lower()

evalscope/benchmarks/data_collection/data_collection_adapter.py

@@ -20,11 +20,12 @@ logger = get_logger()
 @register_benchmark(
     BenchmarkMeta(
         name=DataCollection.NAME,
+        pretty_name='Data-Collection',
         dataset_id='',  # dataset_id need to be set
         description='Custom Data collection, mixing multiple evaluation datasets for '
         'a unified evaluation, aiming to use less data to achieve a more comprehensive '
         'assessment of the model\'s capabilities. '
-        '[Usage Reference](https://evalscope.readthedocs.io/zh-cn/latest/advanced_guides/collection/index.html)',
+        '[Usage Reference](https://evalscope.readthedocs.io/en/latest/advanced_guides/collection/index.html)',
         tags=[Tags.CUSTOM],
         metric_list=['acc'],
         eval_split='test',

evalscope/benchmarks/docvqa/docvqa_adapter.py

@@ -0,0 +1,67 @@
+import json
+from typing import Any, Dict, List
+
+from evalscope.api.benchmark import BenchmarkMeta, VisionLanguageAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.evaluator.state import TaskState
+from evalscope.api.messages import ChatMessageUser, Content, ContentImage, ContentText
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.io_utils import bytes_to_base64
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+PROMPT = """Answer the question according to the image using a single word or phrase.
+{question}
+The last line of your response should be of the form "ANSWER: $ANSWER" (without quotes) where $ANSWER is the answer to the question."""  # noqa: E501
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='docvqa',
+        pretty_name='DocVQA',
+        tags=[Tags.MULTI_MODAL, Tags.KNOWLEDGE, Tags.QA],
+        description=
+        'DocVQA (Document Visual Question Answering) is a benchmark designed to evaluate AI systems on their ability to answer questions based on the content of document images, such as scanned pages, forms, or invoices. Unlike general visual question answering, it requires understanding not just the text extracted by OCR, but also the complex layout, structure, and visual elements of a document.',  # noqa: E501
+        dataset_id='lmms-lab/DocVQA',
+        subset_list=['DocVQA'],
+        metric_list=['anls'],
+        eval_split='validation',
+        prompt_template=PROMPT,
+    )
+)
+class DocVQAAdapter(VisionLanguageAdapter):
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.add_aggregation_name = False
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+
+        input_text = PROMPT.format(question=record['question'])
+        content_list: List[Content] = [ContentText(text=input_text)]
+        image = record.get('image')
+        if image:
+            image_base64 = bytes_to_base64(image['bytes'], format='png', add_header=True)
+            content_list.append(ContentImage(image=image_base64))
+        return Sample(
+            input=[ChatMessageUser(content=content_list)],
+            target=json.dumps(record.get('answers')),  # answers is a list
+            metadata={
+                'questionId': record.get('questionId'),
+                'question_types': record.get('question_types'),
+                'docId': record.get('docId'),
+                'ucsf_document_id': record.get('ucsf_document_id'),
+                'ucsf_document_page_no': record.get('ucsf_document_page_no'),
+            }
+        )
+
+    def extract_answer(self, prediction: str, task_state: TaskState) -> str:
+        import re
+
+        pattern = r'ANSWER:\s*(.*)'
+        match = re.search(pattern, prediction)
+        if match:
+            return match.group(1).strip()
+        return prediction.strip()

evalscope/benchmarks/general_arena/general_arena_adapter.py

@@ -31,7 +31,7 @@ GRADER_TEMPLATE = "<|User Prompt|>\n{question}\n\n<|The Start of Assistant A's A
         'GeneralArena is a custom benchmark designed to evaluate the performance of large language models in a competitive setting, '
         'where models are pitted against each other in custom tasks to determine their relative strengths and weaknesses. You should '
         'provide the model outputs in the format of a list of dictionaries, where each dictionary contains the model name and its report path. '
-        'For detailed instructions on how to use this benchmark, please refer to the [Arena User Guide](https://evalscope.readthedocs.io/zh-cn/latest/user_guides/arena.html).',
+        'For detailed instructions on how to use this benchmark, please refer to the [Arena User Guide](https://evalscope.readthedocs.io/en/latest/user_guides/arena.html).',
         dataset_id='general_arena',
         metric_list=['winrate'],
         few_shot_num=0,

evalscope/benchmarks/general_arena/utils.py

@@ -34,7 +34,8 @@ def process_review_item(review_result: ReviewResult) -> list:
         'Index': str(review_result.index),
         'Input': review_result.input,
         'Question': review_result.input,  # Use input as question
-        'Generated': prediction if prediction != extracted_prediction else extracted_prediction,
+        'Generated':
+        prediction if prediction != extracted_prediction else extracted_prediction or '',  # Ensure no None value
         'Gold': target,
         'Pred': extracted_prediction,
         'Score': sample_score.score.model_dump(exclude_none=True),

evalscope/benchmarks/general_mcq/general_mcq_adapter.py

@@ -20,7 +20,7 @@ logger = get_logger()
         name='general_mcq',
         pretty_name='General-MCQ',
         description='A general multiple-choice question answering dataset for custom evaluation. '
-        'For detailed instructions on how to use this benchmark, please refer to the [User Guide](https://evalscope.readthedocs.io/zh-cn/latest/advanced_guides/custom_dataset/llm.html#mcq).',
+        'For detailed instructions on how to use this benchmark, please refer to the [User Guide](https://evalscope.readthedocs.io/en/latest/advanced_guides/custom_dataset/llm.html#mcq).',
         tags=[Tags.MULTIPLE_CHOICE, Tags.CUSTOM],
         dataset_id='general_mcq',
         subset_list=['default'],

evalscope/benchmarks/general_qa/general_qa_adapter.py

@@ -20,7 +20,7 @@ PROMPT_TEMPLATE = '请回答问题\n{question}'
         name='general_qa',
         pretty_name='General-QA',
         description='A general question answering dataset for custom evaluation. '
-        'For detailed instructions on how to use this benchmark, please refer to the [User Guide](https://evalscope.readthedocs.io/zh-cn/latest/advanced_guides/custom_dataset/llm.html#qa).',  # noqa: E501
+        'For detailed instructions on how to use this benchmark, please refer to the [User Guide](https://evalscope.readthedocs.io/en/latest/advanced_guides/custom_dataset/llm.html#qa).',  # noqa: E501
         tags=[Tags.QA, Tags.CUSTOM],
         dataset_id='general_qa',
         metric_list=['BLEU', 'Rouge'],