evalscope 1.0.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

evalscope/benchmarks/aa_lcr/aa_lcr_adapter.py

@@ -0,0 +1,205 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+# flake8: noqa: E501
+import re
+import urllib.request
+import zipfile
+from pathlib import Path
+from typing import Any, Dict
+
+from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.evaluator import TaskState
+from evalscope.api.messages import ChatMessageUser
+from evalscope.api.metric import Score
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import DEFAULT_EVALSCOPE_CACHE_DIR, Tags
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+# Default judge prompt template
+JUDGE_PROMPT = """Assess whether the following CANDIDATE ANSWER is CORRECT or INCORRECT. For the CANDIDATE ANSWER to be correct, it must be consistent with the OFFICIAL ANSWER.
+
+The question, for reference only: {question}
+The OFFICIAL ANSWER: {correct_answer}
+CANDIDATE ANSWER TO ASSESS: {response}
+
+Reply only with CORRECT or INCORRECT."""
+
+PROMPT_TEMPLATE = """
+BEGIN INPUT DOCUMENTS
+
+{documents_text}
+
+END INPUT DOCUMENTS
+
+Answer the following question using the input documents provided above.
+
+START QUESTION
+
+{question}
+
+END QUESTION
+"""
+
+# New constants for auto-download
+DOWNLOAD_URL: str = (
+    'https://modelscope.cn/datasets/evalscope/AA-LCR/resolve/master/extracted_text/AA-LCR_extracted-text.zip'
+)
+DEFAULT_CACHE_SUBDIR: str = 'aa_lcr'
+DEFAULT_ZIP_NAME: str = 'AA-LCR_extracted-text.zip'
+DEFAULT_EXTRACTED_DIR_NAME: str = 'lcr'
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='aa_lcr',
+        pretty_name='AA-LCR',
+        tags=[Tags.KNOWLEDGE, Tags.REASONING, Tags.LONG_CONTEXT],
+        description='AA-LCR (Artificial Analysis Long Context Retrieval) is a benchmark for evaluating long-context '
+        'retrieval and reasoning capabilities of language models across multiple documents.',  # noqa: E501
+        dataset_id='evalscope/AA-LCR',
+        metric_list=['acc'],
+        few_shot_num=0,
+        train_split=None,
+        eval_split='test',
+        prompt_template=PROMPT_TEMPLATE,
+        extra_params={'text_dir': None}
+    )
+)
+class AALCRAdapter(DefaultDataAdapter):
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        self._use_llm_judge = True
+
+        # Get extra parameters
+        self.text_dir = self.extra_params.get('text_dir')
+
+    def load(self):
+        # Auto download and extract when text_dir is not provided
+        if not self.text_dir:
+            self.text_dir = self._ensure_text_dir_downloaded()
+        elif not Path(self.text_dir).exists():
+            raise ValueError(
+                'AA-LCR text_dir does not exist: '
+                f'{self.text_dir}. Please provide a valid directory or omit text_dir to auto-download.'
+            )
+
+        self.text_dir = Path(self.text_dir)
+        return super().load()
+
+    def _ensure_text_dir_downloaded(self) -> Path:
+        """Ensure AA-LCR extracted texts are available locally; download and extract if missing."""
+        cache_root = Path(DEFAULT_EVALSCOPE_CACHE_DIR) / DEFAULT_CACHE_SUBDIR
+        extracted_dir = cache_root / DEFAULT_EXTRACTED_DIR_NAME
+
+        if extracted_dir.exists():
+            logger.info(f'AA-LCR documents found: {extracted_dir}')
+            return extracted_dir
+
+        cache_root.mkdir(parents=True, exist_ok=True)
+        zip_path = cache_root / DEFAULT_ZIP_NAME
+
+        try:
+            logger.info(f'Downloading AA-LCR documents from {DOWNLOAD_URL} to {zip_path}...')
+            urllib.request.urlretrieve(DOWNLOAD_URL, zip_path)
+
+            logger.info(f'Extracting {zip_path} to {cache_root}...')
+            with zipfile.ZipFile(zip_path, 'r') as zf:
+                zf.extractall(cache_root)
+
+            if not extracted_dir.exists():
+                raise ValueError(f'Extraction succeeded but target directory not found: {extracted_dir}')
+
+            logger.info(f'AA-LCR documents ready at {extracted_dir}')
+            return extracted_dir
+        except Exception as e:
+            raise ValueError(
+                f'Failed to download or extract AA-LCR documents: {e}. '
+                'You can also manually download and set extra_params["text_dir"].'
+            ) from e
+        finally:
+            # Best-effort cleanup of the zip file
+            try:
+                if zip_path.exists():
+                    zip_path.unlink()
+            except Exception:
+                pass
+
+    def _get_context(self, record: Dict[str, Any]) -> str:
+        doc_folder = self.text_dir / record['document_category'] / record['document_set_id']
+
+        # Check if the document folder exists
+        if not doc_folder.exists() or not doc_folder.is_dir():
+            logger.warning(f'Document folder not found: {doc_folder}. Returning empty context.')
+            return ''
+
+        doc_blocks = []
+        try:
+            for file_path in doc_folder.iterdir():
+                if file_path.is_file():
+                    try:
+                        content = file_path.read_text(encoding='utf-8').strip()
+                        if content:
+                            doc_blocks.append(content)
+                    except (IOError, UnicodeDecodeError) as e:
+                        logger.warning(f'Could not read file {file_path}, skipping: {e}')
+        except OSError as e:
+            logger.warning(f'Could not access document folder {doc_folder}: {e}')
+            return f"ERROR: Could not read documents for {record['document_category']}/{record['document_set_id']}"
+
+        documents_text = '\n\n'.join(
+            f'BEGIN DOCUMENT {i + 1}:\n{doc}\nEND DOCUMENT {i + 1}' for i, doc in enumerate(doc_blocks)
+        )
+        return documents_text
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        """Convert a record to a Sample with long-context prompt."""
+        context = self._get_context(record)
+        prompt = self.prompt_template.format(documents_text=context, question=record['question'])
+
+        return Sample(
+            input=[ChatMessageUser(content=prompt)],
+            target=record['answer'],
+            metadata={
+                'question': record['question'],
+                'data_source_urls': record['data_source_urls'],
+                'input_tokens': record.get('input_tokens', 0),
+            }
+        )
+
+    def llm_match_score(
+        self,
+        original_prediction: str,
+        filtered_prediction: str,
+        reference: str,
+        task_state: TaskState,
+    ) -> Score:
+        score = Score(
+            extracted_prediction=filtered_prediction,
+            prediction=original_prediction,
+        )
+
+        judge_prompt = JUDGE_PROMPT.format(
+            question=task_state.metadata['question'], correct_answer=reference, response=filtered_prediction
+        )
+
+        # Request judge and obtain score
+        judge_response = self.llm_judge.judge(prompt=judge_prompt)
+
+        # Parse judge response to get accuracy score
+        # Use word boundaries to avoid matching "CORRECT" within "INCORRECT"
+        is_correct = bool(re.search(r'\bCORRECT\b', judge_response, re.IGNORECASE))
+        score.value = {
+            'acc': 1.0 if is_correct else 0.0,
+        }
+        score.explanation = f'LLM judge: {judge_response}'
+        score.metadata = {
+            'source': 'llm_judge',
+            'judge_strategy': self.judge_strategy,
+            'model': self.llm_judge.model_id,
+        }
+        score.main_score_name = 'acc'
+        return score

evalscope/benchmarks/ai2d/ai2d_adapter.py

@@ -0,0 +1,54 @@
+from typing import Any, Dict, List
+
+from evalscope.api.benchmark import BenchmarkMeta, VisionLanguageAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.evaluator import TaskState
+from evalscope.api.messages import ChatMessageUser, Content, ContentImage, ContentText
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.io_utils import bytes_to_base64
+from evalscope.utils.logger import get_logger
+from evalscope.utils.multi_choices import MultipleChoiceTemplate, parse_answers, prompt
+
+logger = get_logger()
+
+SUBSET_LIST = ['default']
+
+MULT_CHOICE_PROMPT = MultipleChoiceTemplate.SINGLE_ANSWER_COT
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='ai2d',
+        pretty_name='AI2D',
+        tags=[Tags.MULTI_MODAL, Tags.KNOWLEDGE, Tags.QA],
+        description=
+        'AI2D is a benchmark dataset for researching the understanding of diagrams by AI. It contains over 5,000 diverse diagrams from science textbooks (e.g., the water cycle, food webs). Each diagram is accompanied by multiple-choice questions that test an AI\'s ability to interpret visual elements, text labels, and their relationships. The benchmark is challenging because it requires jointly understanding the layout, symbols, and text to answer questions correctly.',  # noqa: E501
+        dataset_id='lmms-lab/ai2d',
+        subset_list=SUBSET_LIST,
+        metric_list=['acc'],
+        eval_split='test',
+        prompt_template=MULT_CHOICE_PROMPT,
+    )
+)
+class Ai2dAdapter(VisionLanguageAdapter):
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        answers_list: list[str] = record['options']
+        input_text = prompt(question=record['question'], choices=answers_list, template=self.prompt_template)
+        content_list: list[Content] = [ContentText(text=input_text)]
+        image = record.get('image')
+        if image:
+            image_base64 = bytes_to_base64(image['bytes'], format='png', add_header=True)
+            content_list.append(ContentImage(image=image_base64))
+
+        label_answer = chr(int(record['answer']) + ord('A'))
+
+        return Sample(input=[ChatMessageUser(content=content_list)], choices=answers_list, target=label_answer)
+
+    def extract_answer(self, prediction: str, task_state: TaskState) -> str:
+        answers = parse_answers(task_state)
+        return ''.join(sorted(list(answers)))

evalscope/benchmarks/aime/aime24_adapter.py

@@ -48,3 +48,8 @@ class AIME24Adapter(DefaultDataAdapter):
                 'solution': record.get('solution', ''),
             },
         )
+
+    def extract_answer(self, prediction: str, task_state):
+        from evalscope.metrics.math_parser import extract_answer
+
+        return extract_answer(prediction)

evalscope/benchmarks/aime/aime25_adapter.py

@@ -1,10 +1,12 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 
+import re
 from typing import Any, Dict
 
 from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter
 from evalscope.api.dataset import Sample
 from evalscope.api.evaluator import TaskState
+from evalscope.api.metric import Score
 from evalscope.api.registry import register_benchmark
 from evalscope.constants import Tags
 from evalscope.utils.logger import get_logger
@@ -13,6 +15,74 @@ from evalscope.utils.logger import get_logger
 
 logger = get_logger()
 
+JUDGE_PROMPT = """
+Look at the following two expressions (answers to a math problem) and judge whether they are equivalent. Only perform trivial simplifications
+
+Examples:
+
+    Expression 1: $2x+3$
+    Expression 2: $3+2x$
+
+Yes
+
+    Expression 1: 3/2
+    Expression 2: 1.5
+
+Yes
+
+    Expression 1: $x^2+2x+1$
+    Expression 2: $y^2+2y+1$
+
+No
+
+    Expression 1: $x^2+2x+1$
+    Expression 2: $(x+1)^2$
+
+Yes
+
+    Expression 1: 3245/5
+    Expression 2: 649
+
+No
+(these are actually equal, don't mark them equivalent if you need to do nontrivial simplifications)
+
+    Expression 1: 2/(-3)
+    Expression 2: -2/3
+
+Yes
+(trivial simplifications are allowed)
+
+    Expression 1: 72 degrees
+    Expression 2: 72
+
+Yes
+(give benefit of the doubt to units)
+
+    Expression 1: 64
+    Expression 2: 64 square feet
+
+Yes
+(give benefit of the doubt to units)
+
+---
+
+YOUR TASK
+
+
+Respond with only "Yes" or "No" (without quotes). Do not include a rationale.
+
+    Expression 1: {expression1}
+    Expression 2: {expression2}
+
+"""
+
+PROMPT_TEMPLATE = """
+Solve the following math problem step by step. Put your answer inside \\boxed{{}}.
+
+{question}
+
+Remember to put your answer inside \\boxed{{}}."""
+
 
 @register_benchmark(
     BenchmarkMeta(
@@ -31,7 +101,7 @@ logger = get_logger()
         few_shot_num=0,
         train_split=None,
         eval_split='test',
-        prompt_template='{question}\nPlease reason step by step, and put your final answer within \\boxed{{}}.',
+        prompt_template=PROMPT_TEMPLATE,
     )
 )
 class AIME25Adapter(DefaultDataAdapter):
@@ -44,3 +114,68 @@ class AIME25Adapter(DefaultDataAdapter):
             input=record['question'],
             target=record['answer'],
         )
+
+    def extract_answer(self, prediction: str, task_state: TaskState) -> str:
+        """
+        Args:
+            prediction (str): The model prediction to extract from
+            task_state (TaskState): The task state for additional context
+
+        Returns:
+            str: The extracted answer
+        """
+        from evalscope.metrics.math_parser import extract_answer
+        from .math_normalize import normalize_answer
+
+        extracted_pred = extract_answer(prediction)
+        filtered_pred = normalize_answer(extracted_pred)
+        return filtered_pred
+
+    def match_score(
+        self, original_prediction: str, filtered_prediction: str, reference: str, task_state: TaskState
+    ) -> Score:
+        from evalscope.metrics.math_parser import extract_answer
+        from .grader import grade_answer
+
+        score = Score(
+            extracted_prediction=filtered_prediction,
+            prediction=original_prediction,
+        )
+
+        # Use the custom grade_answer function for evaluation
+        try:
+            is_correct = grade_answer(extract_answer(original_prediction), reference)
+            accuracy_score = 1.0 if is_correct else 0.0
+            score.value['acc'] = accuracy_score
+        except Exception as e:
+            logger.error(f'Error in custom grading: {e}')
+            score.value['acc'] = 0.0
+            score.metadata['acc'] = f'grading_error: {str(e)}'
+        return score
+
+    def llm_match_score(
+        self, original_prediction: str, filtered_prediction: str, reference: str, task_state: TaskState
+    ) -> Score:
+        score = Score(
+            extracted_prediction=filtered_prediction,
+            prediction=original_prediction,
+        )
+
+        judge_prompt = JUDGE_PROMPT.format(expression1=original_prediction, expression2=reference)
+
+        # Request judge and obtain score
+        judge_response = self.llm_judge.judge(prompt=judge_prompt)
+
+        # Parse judge response to get accuracy score
+        is_correct = bool(re.search(r'\bYes\b', judge_response, re.IGNORECASE))
+        score.value = {
+            'acc': 1.0 if is_correct else 0.0,
+        }
+        score.explanation = f'LLM judge: {judge_response}'
+        score.metadata = {
+            'source': 'llm_judge',
+            'judge_strategy': self.judge_strategy,
+            'model': self.llm_judge.model_id,
+        }
+        score.main_score_name = 'acc'
+        return score