evalscope 0.10.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

evalscope/backend/rag_eval/utils/embedding.py

@@ -1,14 +1,18 @@
 import os
 import torch
 from langchain_core.embeddings import Embeddings
+from langchain_openai.embeddings import OpenAIEmbeddings
+from mteb.encoder_interface import PromptType
 from sentence_transformers import models
 from sentence_transformers.cross_encoder import CrossEncoder
 from sentence_transformers.SentenceTransformer import SentenceTransformer
 from torch import Tensor
+from tqdm import tqdm
 from typing import Dict, List, Optional, Union
 
 from evalscope.backend.rag_eval.utils.tools import download_model
 from evalscope.constants import HubType
+from evalscope.utils.argument_utils import get_supported_params
 from evalscope.utils.logger import get_logger
 
 logger = get_logger()
@@ -18,16 +22,16 @@ class BaseModel(Embeddings):
 
     def __init__(
         self,
-        model_name_or_path: str,
+        model_name_or_path: str = '',
         max_seq_length: int = 512,
-        prompt: str = '',
-        revision: Optional[str] = None,
+        prompt: Optional[str] = None,
+        prompts: Optional[Dict[str, str]] = None,
+        revision: Optional[str] = 'master',
         **kwargs,
     ):
         self.model_name_or_path = model_name_or_path
         self.max_seq_length = max_seq_length
         self.model_kwargs = kwargs.pop('model_kwargs', {})
-        self.model_kwargs['trust_remote_code'] = True
 
         self.config_kwargs = kwargs.pop('config_kwargs', {})
         self.config_kwargs['trust_remote_code'] = True
@@ -36,7 +40,9 @@ class BaseModel(Embeddings):
         self.encode_kwargs['convert_to_tensor'] = True
 
         self.prompt = prompt
+        self.prompts = prompts if prompts else {}
         self.revision = revision
+        self.framework = ['PyTorch']
 
     @property
     def mteb_model_meta(self):
@@ -44,10 +50,22 @@ class BaseModel(Embeddings):
         from mteb import ModelMeta
 
         return ModelMeta(
-            name=os.path.basename(self.model_name_or_path),
+            name='eval/' + os.path.basename(self.model_name_or_path),  # Ensure the name contains a slash
             revision=self.revision,
             languages=None,
             release_date=None,
+            n_parameters=None,
+            memory_usage_mb=None,
+            max_tokens=None,
+            embed_dim=None,
+            license=None,
+            open_weights=None,
+            public_training_code=None,
+            public_training_data=None,
+            similarity_fn_name=None,
+            use_instructions=None,
+            training_datasets=None,
+            framework=self.framework,
         )
 
     def embed_documents(self, texts: List[str]) -> List[List[float]]:
@@ -59,7 +77,7 @@ class BaseModel(Embeddings):
         Returns:
             List of embeddings.
         """
-        return self.encode_corpus(texts).tolist()
+        return self.encode(texts).tolist()
 
     def embed_query(self, text: str) -> List[float]:
         """Embed query text. Compact langchain.
@@ -70,19 +88,17 @@ class BaseModel(Embeddings):
         Returns:
             Embedding.
         """
-        return self.encode_queries(text).tolist()
+        return self.encode(text).tolist()
 
     def encode(self, texts: Union[str, List[str]], **kwargs) -> List[List[float]]:
         """Embed text."""
         raise NotImplementedError
 
-    def encode_queries(self, queries: List[str], **kwargs) -> list[torch.Tensor]:
-        """Embed query text. Compact mteb."""
-        raise NotImplementedError
-
-    def encode_corpus(self, corpus: Union[List[str], List[Dict[str, str]]], **kwargs) -> list[torch.Tensor]:
-        """Embed search docs . Compact mteb."""
-        raise NotImplementedError
+    def get_prompt(self, task_name: str) -> Optional[str]:
+        """Get prompt for the given task name."""
+        if self.prompt:
+            return self.prompt
+        return self.prompts.get(task_name, None)
 
 
 class SentenceTransformerModel(BaseModel):
@@ -90,6 +106,9 @@ class SentenceTransformerModel(BaseModel):
     def __init__(self, model_name_or_path: str, pooling_mode: Optional[str] = None, **kwargs):
         super().__init__(model_name_or_path, **kwargs)
 
+        self.framework = ['Sentence Transformers', 'PyTorch']
+
+        self.model_kwargs['trust_remote_code'] = True
         if not pooling_mode:
             self.model = SentenceTransformer(
                 self.model_name_or_path,
@@ -110,43 +129,59 @@ class SentenceTransformerModel(BaseModel):
 
         self.model.max_seq_length = self.max_seq_length
 
-    def encode(self, texts: Union[str, List[str]], prompt=None, **kwargs) -> List[torch.Tensor]:
-        kwargs.pop('prompt_name', '')  # remove prompt name, use prompt
+        self.supported_encode_params = get_supported_params(self.model.encode)
+
+    def encode(self, texts: Union[str, List[str]], **kwargs) -> List[torch.Tensor]:
+        # pop unused kwargs
+        extra_params = {}
+        for key in list(kwargs.keys()):
+            if key not in self.supported_encode_params:
+                extra_params[key] = kwargs.pop(key)
         self.encode_kwargs.update(kwargs)
 
+        # set prompt if provided
+        prompt = None
+        prompt_type = extra_params.pop('prompt_type', '')
+        task_name = extra_params.pop('task_name', '')
+        if prompt_type and prompt_type == PromptType.query:
+            prompt = self.get_prompt(task_name)
+
         embeddings = self.model.encode(texts, prompt=prompt, **self.encode_kwargs)
         assert isinstance(embeddings, Tensor)
         return embeddings.cpu().detach()
 
-    def encode_queries(self, queries, **kwargs):
-        return self.encode(queries, prompt=self.prompt)
-
-    def encode_corpus(self, corpus, **kwargs):
-        if isinstance(corpus[0], dict):
-            input_texts = ['{} {}'.format(doc.get('title', ''), doc['text']).strip() for doc in corpus]
-        else:
-            input_texts = corpus
-        return self.encode(input_texts)
-
 
 class CrossEncoderModel(BaseModel):
 
     def __init__(self, model_name_or_path: str, **kwargs):
         super().__init__(model_name_or_path, **kwargs)
+
+        self.framework = ['Sentence Transformers', 'PyTorch']
+
         self.model = CrossEncoder(
             self.model_name_or_path,
             trust_remote_code=True,
             max_length=self.max_seq_length,
+            automodel_args=self.model_kwargs,
         )
-
-    def predict(self, sentences: List[List[str]], **kwargs) -> List[List[float]]:
+        self.tokenizer = self.model.tokenizer
+        # set pad token
+        if self.tokenizer.pad_token is None:
+            self.tokenizer.pad_token = self.tokenizer.eos_token
+        if ('pad_token_id' not in self.model.config) or (self.model.config.pad_token_id is None):
+            self.model.config.update({'pad_token_id': self.tokenizer.eos_token_id})
+
+        self.supported_encode_params = get_supported_params(self.model.predict)
+
+    def predict(self, sentences: List[List[str]], **kwargs) -> Tensor:
+        for key in list(kwargs.keys()):
+            if key not in self.supported_encode_params:
+                kwargs.pop(key)
         self.encode_kwargs.update(kwargs)
 
-        if len(sentences[0]) == 3:  # Note: For mteb retrieval task
+        if len(sentences[0]) == 2:  # Note: For mteb retrieval task
             processed_sentences = []
-            for query, docs, instruction in sentences:
-                if isinstance(docs, dict):
-                    docs = docs['text']
+            for query, docs in sentences:
                 processed_sentences.append((self.prompt + query, docs))
             sentences = processed_sentences
         embeddings = self.model.predict(sentences, **self.encode_kwargs)
@@ -154,6 +189,60 @@ class CrossEncoderModel(BaseModel):
         return embeddings
 
 
+class APIEmbeddingModel(BaseModel):
+
+    def __init__(self, **kwargs):
+        self.model_name = kwargs.get('model_name')
+        self.openai_api_base = kwargs.get('api_base')
+        self.openai_api_key = kwargs.get('api_key')
+        self.dimensions = kwargs.get('dimensions')
+        self.check_embedding_ctx_length = kwargs.get('check_embedding_ctx_length', False)
+        self.framework = ['API']
+
+        self.model = OpenAIEmbeddings(
+            model=self.model_name,
+            openai_api_base=self.openai_api_base,
+            openai_api_key=self.openai_api_key,
+            dimensions=self.dimensions,
+            check_embedding_ctx_length=self.check_embedding_ctx_length,
+        )
+
+        super().__init__(model_name_or_path=self.model_name, **kwargs)
+
+        self.batch_size = self.encode_kwargs.get('batch_size', 10)
+
+        self.supported_encode_params = get_supported_params(self.model.embed_documents)
+
+    def encode(self, texts: Union[str, List[str]], **kwargs) -> Tensor:
+        # pop unused kwargs
+        extra_params = {}
+        for key in list(kwargs.keys()):
+            if key not in self.supported_encode_params:
+                extra_params[key] = kwargs.pop(key)
+        self.encode_kwargs.update(kwargs)
+
+        # set prompt if provided
+        prompt = None
+        prompt_type = extra_params.pop('prompt_type', '')
+        task_name = extra_params.pop('task_name', '')
+        if prompt_type and prompt_type == PromptType.query:
+            prompt = self.get_prompt(task_name)
+
+        if isinstance(texts, str):
+            texts = [texts]
+
+        embeddings: List[List[float]] = []
+        for i in tqdm(range(0, len(texts), self.batch_size)):
+            # set prompt if provided
+            if prompt is not None:
+                batch_texts = [prompt + text for text in texts[i:i + self.batch_size]]
+            else:
+                batch_texts = texts[i:i + self.batch_size]
+            response = self.model.embed_documents(batch_texts, chunk_size=self.batch_size)
+            embeddings.extend(response)
+        return torch.tensor(embeddings)
+
+
 class EmbeddingModel:
     """Custom embeddings"""
 
@@ -165,6 +254,10 @@ class EmbeddingModel:
         revision: Optional[str] = 'master',
         **kwargs,
     ):
+        if kwargs.get('model_name'):
+            # If model_name is provided, use OpenAIEmbeddings
+            return APIEmbeddingModel(**kwargs)
+
         # If model path does not exist and hub is 'modelscope', download the model
         if not os.path.exists(model_name_or_path) and hub == HubType.MODELSCOPE:
             model_name_or_path = download_model(model_name_or_path, revision)

evalscope/backend/rag_eval/utils/llm.py

@@ -2,11 +2,10 @@ import os
 from langchain_core.callbacks.manager import CallbackManagerForLLMRun
 from langchain_core.language_models.llms import LLM as BaseLLM
 from langchain_openai import ChatOpenAI
-from modelscope.utils.hf_util import GenerationConfig
 from typing import Any, Dict, Iterator, List, Mapping, Optional
 
-from evalscope.constants import DEFAULT_MODEL_REVISION
-from evalscope.models import ChatGenerationModelAdapter
+from evalscope.api.model import GenerateConfig, Model, get_model
+from evalscope.constants import DEFAULT_MODEL_REVISION, EvalType
 
 
 class LLM:
@@ -16,9 +15,9 @@ class LLM:
         api_base = kw.get('api_base', None)
         if api_base:
             return ChatOpenAI(
-                model_name=kw.get('model_name', ''),
-                openai_api_base=api_base,
-                openai_api_key=kw.get('api_key', 'EMPTY'),
+                model=kw.get('model_name', ''),
+                base_url=api_base,
+                api_key=kw.get('api_key', 'EMPTY'),
             )
         else:
             return LocalLLM(**kw)
@@ -30,17 +29,19 @@ class LocalLLM(BaseLLM):
     model_name_or_path: str
     model_revision: str = DEFAULT_MODEL_REVISION
     template_type: Optional[str] = None
-    model_name: Optional[str]
-    model: Optional[ChatGenerationModelAdapter]
-    generation_config: Optional[Dict]
+    model_name: Optional[str] = None
+    model: Optional[Model] = None
+    generation_config: Optional[Dict] = {}
 
     def __init__(self, **kw):
         super().__init__(**kw)
         self.model_name = os.path.basename(self.model_name_or_path)
-        self.model = ChatGenerationModelAdapter(
-            model_id=self.model_name_or_path,
-            model_revision=self.model_revision,
-            generation_config=GenerationConfig(**self.generation_config) if self.generation_config else None,
+
+        # Create and initialize the local model
+        self.model = get_model(
+            model=self.model_name_or_path,
+            eval_type=EvalType.CHECKPOINT,
+            config=GenerateConfig(**self.generation_config),
         )
 
     def _call(
@@ -51,10 +52,9 @@ class LocalLLM(BaseLLM):
         **kwargs: Any,
     ) -> str:
         """Run the LLM on the given input."""
-        infer_cfg = {'stop': stop}
 
-        response = self.model._model_generate(prompt, infer_cfg)
-        return response
+        response = self.model.generate(input=prompt)
+        return response.completion
 
     @property
     def _identifying_params(self) -> Dict[str, Any]:

evalscope/backend/vlm_eval_kit/backend_manager.py

@@ -1,10 +1,12 @@
 import copy
+import os
 import subprocess
 from functools import partial
 from typing import Optional, Union
 
 from evalscope.backend.base import BackendManager
-from evalscope.utils import get_valid_list, is_module_installed
+from evalscope.utils.import_utils import is_module_installed
+from evalscope.utils.io_utils import get_valid_list
 from evalscope.utils.logger import get_logger
 
 logger = get_logger()
@@ -66,8 +68,11 @@ class VLMEvalKitBackendManager(BackendManager):
                     del remain_cfg['name']  # remove not used args
                     del remain_cfg['type']  # remove not used args
 
-                    self.valid_models.update({model_type: partial(model_class, model=model_type, **remain_cfg)})
-                    new_model_names.append(model_type)
+                    norm_model_type = os.path.basename(model_type).replace(':', '-').replace('.', '_')
+                    model_cfg['type'] = norm_model_type
+
+                    self.valid_models.update({norm_model_type: partial(model_class, model=model_type, **remain_cfg)})
+                    new_model_names.append(norm_model_type)
                 else:
                     remain_cfg = copy.deepcopy(model_cfg)
                     del remain_cfg['name']  # remove not used args

evalscope/benchmarks/__init__.py

@@ -2,16 +2,17 @@
 import glob
 import importlib
 import os
+import time
 
-from evalscope.benchmarks.benchmark import Benchmark, BenchmarkMeta
-from evalscope.benchmarks.data_adapter import DataAdapter
 from evalscope.utils import get_logger
 
 logger = get_logger()
 
 # Using glob to find all files matching the pattern
-pattern = os.path.join(os.path.dirname(__file__), '*', '*_adapter.py')
-files = glob.glob(pattern, recursive=False)
+pattern = os.path.join(os.path.dirname(__file__), '*', '**', '*_adapter.py')
+files = glob.glob(pattern, recursive=True)
+
+import_times = []
 
 for file_path in files:
     if file_path.endswith('.py') and not os.path.basename(file_path).startswith('_'):
@@ -19,5 +20,16 @@ for file_path in files:
         relative_path = os.path.relpath(file_path, os.path.dirname(__file__))
         module_path = relative_path[:-3].replace(os.path.sep, '.')  # strip '.py' and convert to module path
         full_path = f'evalscope.benchmarks.{module_path}'
+
+        start_time = time.perf_counter()
         importlib.import_module(full_path)
-        # print(f'Importing {full_path}')
+        end_time = time.perf_counter()
+
+        import_times.append((full_path, end_time - start_time))
+
+# Sort by import time in descending order
+import_times.sort(key=lambda x: x[1], reverse=True)
+
+# Log the sorted import times
+for module, duration in import_times:
+    logger.debug(f'Module {module} imported in {duration:.6f} seconds')

evalscope/benchmarks/aa_lcr/aa_lcr_adapter.py

@@ -0,0 +1,205 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+# flake8: noqa: E501
+import re
+import urllib.request
+import zipfile
+from pathlib import Path
+from typing import Any, Dict
+
+from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.evaluator import TaskState
+from evalscope.api.messages import ChatMessageUser
+from evalscope.api.metric import Score
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import DEFAULT_EVALSCOPE_CACHE_DIR, Tags
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+# Default judge prompt template
+JUDGE_PROMPT = """Assess whether the following CANDIDATE ANSWER is CORRECT or INCORRECT. For the CANDIDATE ANSWER to be correct, it must be consistent with the OFFICIAL ANSWER.
+
+The question, for reference only: {question}
+The OFFICIAL ANSWER: {correct_answer}
+CANDIDATE ANSWER TO ASSESS: {response}
+
+Reply only with CORRECT or INCORRECT."""
+
+PROMPT_TEMPLATE = """
+BEGIN INPUT DOCUMENTS
+
+{documents_text}
+
+END INPUT DOCUMENTS
+
+Answer the following question using the input documents provided above.
+
+START QUESTION
+
+{question}
+
+END QUESTION
+"""
+
+# New constants for auto-download
+DOWNLOAD_URL: str = (
+    'https://modelscope.cn/datasets/evalscope/AA-LCR/resolve/master/extracted_text/AA-LCR_extracted-text.zip'
+)
+DEFAULT_CACHE_SUBDIR: str = 'aa_lcr'
+DEFAULT_ZIP_NAME: str = 'AA-LCR_extracted-text.zip'
+DEFAULT_EXTRACTED_DIR_NAME: str = 'lcr'
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='aa_lcr',
+        pretty_name='AA-LCR',
+        tags=[Tags.KNOWLEDGE, Tags.REASONING, Tags.LONG_CONTEXT],
+        description='AA-LCR (Artificial Analysis Long Context Retrieval) is a benchmark for evaluating long-context '
+        'retrieval and reasoning capabilities of language models across multiple documents.',  # noqa: E501
+        dataset_id='evalscope/AA-LCR',
+        metric_list=['acc'],
+        few_shot_num=0,
+        train_split=None,
+        eval_split='test',
+        prompt_template=PROMPT_TEMPLATE,
+        extra_params={'text_dir': None}
+    )
+)
+class AALCRAdapter(DefaultDataAdapter):
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        self._use_llm_judge = True
+
+        # Get extra parameters
+        self.text_dir = self.extra_params.get('text_dir')
+
+    def load(self):
+        # Auto download and extract when text_dir is not provided
+        if not self.text_dir:
+            self.text_dir = self._ensure_text_dir_downloaded()
+        elif not Path(self.text_dir).exists():
+            raise ValueError(
+                'AA-LCR text_dir does not exist: '
+                f'{self.text_dir}. Please provide a valid directory or omit text_dir to auto-download.'
+            )
+
+        self.text_dir = Path(self.text_dir)
+        return super().load()
+
+    def _ensure_text_dir_downloaded(self) -> Path:
+        """Ensure AA-LCR extracted texts are available locally; download and extract if missing."""
+        cache_root = Path(DEFAULT_EVALSCOPE_CACHE_DIR) / DEFAULT_CACHE_SUBDIR
+        extracted_dir = cache_root / DEFAULT_EXTRACTED_DIR_NAME
+
+        if extracted_dir.exists():
+            logger.info(f'AA-LCR documents found: {extracted_dir}')
+            return extracted_dir
+
+        cache_root.mkdir(parents=True, exist_ok=True)
+        zip_path = cache_root / DEFAULT_ZIP_NAME
+
+        try:
+            logger.info(f'Downloading AA-LCR documents from {DOWNLOAD_URL} to {zip_path}...')
+            urllib.request.urlretrieve(DOWNLOAD_URL, zip_path)
+
+            logger.info(f'Extracting {zip_path} to {cache_root}...')
+            with zipfile.ZipFile(zip_path, 'r') as zf:
+                zf.extractall(cache_root)
+
+            if not extracted_dir.exists():
+                raise ValueError(f'Extraction succeeded but target directory not found: {extracted_dir}')
+
+            logger.info(f'AA-LCR documents ready at {extracted_dir}')
+            return extracted_dir
+        except Exception as e:
+            raise ValueError(
+                f'Failed to download or extract AA-LCR documents: {e}. '
+                'You can also manually download and set extra_params["text_dir"].'
+            ) from e
+        finally:
+            # Best-effort cleanup of the zip file
+            try:
+                if zip_path.exists():
+                    zip_path.unlink()
+            except Exception:
+                pass
+
+    def _get_context(self, record: Dict[str, Any]) -> str:
+        doc_folder = self.text_dir / record['document_category'] / record['document_set_id']
+
+        # Check if the document folder exists
+        if not doc_folder.exists() or not doc_folder.is_dir():
+            logger.warning(f'Document folder not found: {doc_folder}. Returning empty context.')
+            return ''
+
+        doc_blocks = []
+        try:
+            for file_path in doc_folder.iterdir():
+                if file_path.is_file():
+                    try:
+                        content = file_path.read_text(encoding='utf-8').strip()
+                        if content:
+                            doc_blocks.append(content)
+                    except (IOError, UnicodeDecodeError) as e:
+                        logger.warning(f'Could not read file {file_path}, skipping: {e}')
+        except OSError as e:
+            logger.warning(f'Could not access document folder {doc_folder}: {e}')
+            return f"ERROR: Could not read documents for {record['document_category']}/{record['document_set_id']}"
+
+        documents_text = '\n\n'.join(
+            f'BEGIN DOCUMENT {i + 1}:\n{doc}\nEND DOCUMENT {i + 1}' for i, doc in enumerate(doc_blocks)
+        )
+        return documents_text
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        """Convert a record to a Sample with long-context prompt."""
+        context = self._get_context(record)
+        prompt = self.prompt_template.format(documents_text=context, question=record['question'])
+
+        return Sample(
+            input=[ChatMessageUser(content=prompt)],
+            target=record['answer'],
+            metadata={
+                'question': record['question'],
+                'data_source_urls': record['data_source_urls'],
+                'input_tokens': record.get('input_tokens', 0),
+            }
+        )
+
+    def llm_match_score(
+        self,
+        original_prediction: str,
+        filtered_prediction: str,
+        reference: str,
+        task_state: TaskState,
+    ) -> Score:
+        score = Score(
+            extracted_prediction=filtered_prediction,
+            prediction=original_prediction,
+        )
+
+        judge_prompt = JUDGE_PROMPT.format(
+            question=task_state.metadata['question'], correct_answer=reference, response=filtered_prediction
+        )
+
+        # Request judge and obtain score
+        judge_response = self.llm_judge.judge(prompt=judge_prompt)
+
+        # Parse judge response to get accuracy score
+        # Use word boundaries to avoid matching "CORRECT" within "INCORRECT"
+        is_correct = bool(re.search(r'\bCORRECT\b', judge_response, re.IGNORECASE))
+        score.value = {
+            'acc': 1.0 if is_correct else 0.0,
+        }
+        score.explanation = f'LLM judge: {judge_response}'
+        score.metadata = {
+            'source': 'llm_judge',
+            'judge_strategy': self.judge_strategy,
+            'model': self.llm_judge.model_id,
+        }
+        score.main_score_name = 'acc'
+        return score

evalscope/benchmarks/ai2d/ai2d_adapter.py

@@ -0,0 +1,54 @@
+from typing import Any, Dict, List
+
+from evalscope.api.benchmark import BenchmarkMeta, VisionLanguageAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.evaluator import TaskState
+from evalscope.api.messages import ChatMessageUser, Content, ContentImage, ContentText
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.io_utils import bytes_to_base64
+from evalscope.utils.logger import get_logger
+from evalscope.utils.multi_choices import MultipleChoiceTemplate, parse_answers, prompt
+
+logger = get_logger()
+
+SUBSET_LIST = ['default']
+
+MULT_CHOICE_PROMPT = MultipleChoiceTemplate.SINGLE_ANSWER_COT
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='ai2d',
+        pretty_name='AI2D',
+        tags=[Tags.MULTI_MODAL, Tags.KNOWLEDGE, Tags.QA],
+        description=
+        'AI2D is a benchmark dataset for researching the understanding of diagrams by AI. It contains over 5,000 diverse diagrams from science textbooks (e.g., the water cycle, food webs). Each diagram is accompanied by multiple-choice questions that test an AI\'s ability to interpret visual elements, text labels, and their relationships. The benchmark is challenging because it requires jointly understanding the layout, symbols, and text to answer questions correctly.',  # noqa: E501
+        dataset_id='lmms-lab/ai2d',
+        subset_list=SUBSET_LIST,
+        metric_list=['acc'],
+        eval_split='test',
+        prompt_template=MULT_CHOICE_PROMPT,
+    )
+)
+class Ai2dAdapter(VisionLanguageAdapter):
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        answers_list: list[str] = record['options']
+        input_text = prompt(question=record['question'], choices=answers_list, template=self.prompt_template)
+        content_list: list[Content] = [ContentText(text=input_text)]
+        image = record.get('image')
+        if image:
+            image_base64 = bytes_to_base64(image['bytes'], format='png', add_header=True)
+            content_list.append(ContentImage(image=image_base64))
+
+        label_answer = chr(int(record['answer']) + ord('A'))
+
+        return Sample(input=[ChatMessageUser(content=content_list)], choices=answers_list, target=label_answer)
+
+    def extract_answer(self, prediction: str, task_state: TaskState) -> str:
+        answers = parse_answers(task_state)
+        return ''.join(sorted(list(answers)))