evalscope 0.7.1__py3-none-any.whl → 0.8.0__py3-none-any.whl

evalscope/backend/rag_eval/ragas/tasks/testset_generation.py

@@ -1,6 +1,4 @@
-import asyncio
 import os
-
 import pandas as pd
 from ragas.embeddings import LangchainEmbeddingsWrapper
 from ragas.llms import LangchainLLMWrapper
@@ -9,117 +7,12 @@ from tqdm import tqdm
 from evalscope.backend.rag_eval import LLM, ChatOpenAI, EmbeddingModel
 from evalscope.backend.rag_eval.ragas.arguments import TestsetGenerationArguments
 from evalscope.utils.logger import get_logger
-from .translate_prompt import translate_prompts
+from .build_distribution import default_query_distribution
+from .build_transform import default_transforms
 
 logger = get_logger()
 
 
-def get_transform(llm, embedding, language):
-    """
-    Creates and returns a default set of transforms for processing a knowledge graph.
-    """
-    from ragas.testset.transforms.engine import Parallel
-    from ragas.testset.transforms.extractors import (
-        EmbeddingExtractor,
-        HeadlinesExtractor,
-        SummaryExtractor,
-    )
-    from ragas.testset.transforms.extractors.llm_based import NERExtractor, ThemesExtractor
-    from ragas.testset.transforms.relationship_builders import (
-        CosineSimilarityBuilder,
-        OverlapScoreBuilder,
-    )
-    from ragas.testset.transforms.splitters import HeadlineSplitter
-    from ragas.testset.transforms.filters import CustomNodeFilter
-    from ragas.testset.graph import NodeType
-    from ragas.utils import num_tokens_from_string
-
-    def summary_filter(node):
-        return (node.type == NodeType.DOCUMENT and num_tokens_from_string(node.properties['page_content']) > 500)
-
-    summary_extractor = SummaryExtractor(llm=llm, filter_nodes=lambda node: summary_filter(node))
-    ner_extractor = NERExtractor(llm=llm, filter_nodes=lambda node: node.type == NodeType.CHUNK)
-    theme_extractor = ThemesExtractor(llm=llm)
-    headline_extractor = HeadlinesExtractor(llm=llm)
-
-    asyncio.run(
-        translate_prompts(
-            prompts=[
-                summary_extractor,
-                theme_extractor,
-                ner_extractor,
-                headline_extractor,
-            ],
-            target_lang=language,
-            llm=llm,
-            adapt_instruction=True,
-        ))
-
-    splitter = HeadlineSplitter(min_tokens=500)
-
-    summary_emb_extractor = EmbeddingExtractor(
-        embedding_model=embedding,
-        property_name='summary_embedding',
-        embed_property_name='summary',
-        filter_nodes=lambda node: summary_filter(node),
-    )
-
-    cosine_sim_builder = CosineSimilarityBuilder(
-        property_name='summary_embedding',
-        new_property_name='summary_similarity',
-        threshold=0.7,
-        filter_nodes=lambda node: summary_filter(node),
-    )
-
-    ner_overlap_sim = OverlapScoreBuilder(threshold=0.01, filter_nodes=lambda node: node.type == NodeType.CHUNK)
-
-    node_filter = CustomNodeFilter(llm=llm, filter_nodes=lambda node: node.type == NodeType.CHUNK)
-
-    transforms = [
-        headline_extractor,
-        splitter,
-        summary_extractor,
-        node_filter,
-        Parallel(summary_emb_extractor, theme_extractor, ner_extractor),
-        Parallel(cosine_sim_builder, ner_overlap_sim),
-    ]
-
-    return transforms
-
-
-def get_distribution(llm, distribution, language):
-    from ragas.testset.synthesizers.multi_hop import (
-        MultiHopAbstractQuerySynthesizer,
-        MultiHopSpecificQuerySynthesizer,
-    )
-    from ragas.testset.synthesizers.single_hop.specific import (
-        SingleHopSpecificQuerySynthesizer, )
-
-    single_hop = SingleHopSpecificQuerySynthesizer(llm=llm)
-    multi_hop_abs = MultiHopAbstractQuerySynthesizer(llm=llm)
-    multi_hop_spec = MultiHopSpecificQuerySynthesizer(llm=llm)
-
-    asyncio.run(
-        translate_prompts(
-            prompts=[
-                single_hop,
-                multi_hop_abs,
-                multi_hop_spec,
-            ],
-            target_lang=language,
-            llm=llm,
-            adapt_instruction=True,
-        ))
-
-    mapping = {
-        'simple': single_hop,
-        'multi_context': multi_hop_abs,
-        'reasoning': multi_hop_spec,
-    }
-
-    return [(mapping[key], distribution[key]) for key in mapping if key in distribution]
-
-
 def get_knowledge_graph(documents, transforms, local_file, run_config):
     from ragas.testset.graph import KnowledgeGraph, Node, NodeType
     from ragas.testset.transforms import apply_transforms
@@ -153,15 +46,9 @@ def get_knowledge_graph(documents, transforms, local_file, run_config):
 
 
 def get_persona(llm, kg, language):
-    from evalscope.backend.rag_eval.ragas.prompts.persona_prompt import PersonaGenerationPromptZH
-    from ragas.testset.persona import generate_personas_from_kg, PersonaGenerationPrompt
-    from ragas.testset.graph import Node
+    from ragas.testset.persona import PersonaGenerationPrompt, generate_personas_from_kg
 
-    def filter(node: Node) -> bool:
-        if (node.type.name == 'DOCUMENT' and node.properties.get('summary_embedding') is not None):
-            return True
-        else:
-            return False
+    from evalscope.backend.rag_eval.ragas.prompts.persona_prompt import PersonaGenerationPromptZH
 
     if language == 'chinese':
         persona_prompt = PersonaGenerationPromptZH()
@@ -176,27 +63,21 @@ def get_persona(llm, kg, language):
     #         adapt_instruction=True,
     #     ))
 
-    return generate_personas_from_kg(
-        llm=llm,
-        kg=kg,
-        num_personas=3,
-        persona_generation_prompt=persona_prompt,
-        filter_fn=filter,
-    )
+    return generate_personas_from_kg(llm=llm, kg=kg, num_personas=3, persona_generation_prompt=persona_prompt)
 
 
 def load_data(file_path):
     from langchain_community.document_loaders import UnstructuredFileLoader
 
-    loader = UnstructuredFileLoader(file_path, mode='elements')
+    loader = UnstructuredFileLoader(file_path, mode='single')
     data = loader.load()
     return data
 
 
 def generate_testset(args: TestsetGenerationArguments) -> None:
 
-    from ragas.testset import TestsetGenerator
     from ragas import RunConfig
+    from ragas.testset import TestsetGenerator
 
     # load data
     documents = load_data(args.docs)
@@ -208,23 +89,26 @@ def generate_testset(args: TestsetGenerationArguments) -> None:
     wrapped_llm = LangchainLLMWrapper(generator_llm)
     wrapped_embeddings = LangchainEmbeddingsWrapper(embeddings)
 
-    # Change resulting question type distribution
-    distributions = get_distribution(wrapped_llm, args.distribution, args.language)
-
-    run_config = RunConfig(timeout=600, max_retries=3, max_wait=120, max_workers=1, log_tenacity=True)
     # get transforms
-    transforms = get_transform(
+    transforms = default_transforms(
+        documents,
         wrapped_llm,
         wrapped_embeddings,
         args.language,
     )
 
+    run_config = RunConfig(timeout=600, max_retries=10, max_wait=120, max_workers=1, log_tenacity=True)
     # get knowledge graph
     knowledge_graph = get_knowledge_graph(documents, transforms, args.knowledge_graph, run_config)
-
+    # get persona
     persona_list = get_persona(llm=wrapped_llm, kg=knowledge_graph, language=args.language)
 
-    generator = TestsetGenerator(llm=wrapped_llm, knowledge_graph=knowledge_graph, persona_list=persona_list)
+    # Change resulting question type distribution
+    distributions = default_query_distribution(wrapped_llm, knowledge_graph, args.language)
+
+    # generate testset
+    generator = TestsetGenerator(
+        llm=wrapped_llm, embedding_model=wrapped_embeddings, knowledge_graph=knowledge_graph, persona_list=persona_list)
 
     testset = generator.generate(
         testset_size=args.test_size,

evalscope/backend/rag_eval/ragas/tasks/translate_prompt.py

@@ -1,11 +1,11 @@
-import os
 import asyncio
-from typing import List
-from ragas.prompt import PromptMixin
+import os
 from ragas.llms import BaseRagasLLM
+from ragas.prompt import PromptMixin, PydanticPrompt
 from ragas.utils import RAGAS_SUPPORTED_LANGUAGE_CODES
-from evalscope.utils.logger import get_logger
+from typing import List
 
+from evalscope.utils.logger import get_logger
 
 logger = get_logger()
 
@@ -17,9 +17,7 @@ async def translate_prompt(
     adapt_instruction: bool = False,
 ):
     if target_lang not in RAGAS_SUPPORTED_LANGUAGE_CODES:
-        logger.warning(
-            f'{target_lang} is not in supported language: {list(RAGAS_SUPPORTED_LANGUAGE_CODES)}'
-        )
+        logger.warning(f'{target_lang} is not in supported language: {list(RAGAS_SUPPORTED_LANGUAGE_CODES)}')
         return
 
     if not issubclass(type(prompt_user), PromptMixin):
@@ -28,9 +26,7 @@ async def translate_prompt(
 
     class_name = prompt_user.__class__.__name__
     current_dir = os.path.dirname(__file__)
-    prompt_dir = os.path.abspath(
-        os.path.join(current_dir, f'../prompts/{target_lang}/{class_name}')
-    )
+    prompt_dir = os.path.abspath(os.path.join(current_dir, f'../prompts/{target_lang}/{class_name}'))
     os.makedirs(prompt_dir, exist_ok=True)
 
     try:
@@ -43,8 +39,7 @@ async def translate_prompt(
 
     logger.info(f'Translating prompts to {target_lang}')
     adapted_prompts = await prompt_user.adapt_prompts(
-        language=target_lang, llm=llm, adapt_instruction=adapt_instruction
-    )
+        language=target_lang, llm=llm, adapt_instruction=adapt_instruction)
     prompt_user.set_prompts(**adapted_prompts)
     try:
         prompt_user.save_prompts(prompt_dir)
@@ -62,11 +57,6 @@ async def translate_prompts(
     adapt_instruction: bool = False,
 ):
     if target_lang and target_lang != 'english':
-        await asyncio.gather(
-            *[
-                translate_prompt(prompt, target_lang, llm, adapt_instruction)
-                for prompt in prompts
-            ]
-        )
+        await asyncio.gather(*[translate_prompt(prompt, target_lang, llm, adapt_instruction) for prompt in prompts])
 
         logger.info('Translate prompts finished')

evalscope/backend/rag_eval/utils/clip.py

@@ -1,33 +1,37 @@
 import os
 import torch
 import torch.nn.functional as F
-from typing import List
+from langchain_core.embeddings import Embeddings
 from PIL import Image
-from evalscope.backend.rag_eval.utils.tools import download_model, PIL_to_base64
 from transformers import AutoModel, AutoProcessor
-from langchain_core.embeddings import Embeddings
+from typing import List
+
+from evalscope.backend.rag_eval.utils.tools import PIL_to_base64, download_model
+from evalscope.constants import HubType
 
 
 class VisionModel:
+
     @staticmethod
     def load(**kw):
-        api_base = kw.get("api_base", None)
+        api_base = kw.get('api_base', None)
         if api_base:
 
             return VLMAPI(
-                model_name=kw.get("model_name", ""),
+                model_name=kw.get('model_name', ''),
                 openai_api_base=api_base,
-                openai_api_key=kw.get("api_key", "EMPTY"),
-                prompt=kw.get("prompt", None),
+                openai_api_key=kw.get('api_key', 'EMPTY'),
+                prompt=kw.get('prompt', None),
             )
         else:
             return CLIPModel(**kw)
 
 
 class VLMAPI:
+
     def __init__(self, model_name, openai_api_base, openai_api_key, prompt=None):
-        from langchain_openai import ChatOpenAI
         from langchain_core.prompts import ChatPromptTemplate
+        from langchain_openai import ChatOpenAI
 
         self.model_name = model_name
         self.model = ChatOpenAI(
@@ -35,46 +39,45 @@ class VLMAPI:
             openai_api_base=openai_api_base,
             openai_api_key=openai_api_key,
         )
-        self.default_prompt = "Please describe this image in general. Directly provide the description, do not include prefix like 'This image depicts'"
-        self.prompt = ChatPromptTemplate.from_messages(
-            [
-                ("system", prompt if prompt else self.default_prompt),
-                (
-                    "user",
-                    [
-                        {
-                            "type": "image_url",
-                            "image_url": {"url": "data:image/jpeg;base64,{image_data}"},
-                        }
-                    ],
-                ),
-            ]
-        )
+        self.default_prompt = "Please describe this image in general. Directly provide the description, do not include prefix like 'This image depicts'"  # noqa: E501
+        self.prompt = ChatPromptTemplate.from_messages([
+            ('system', prompt if prompt else self.default_prompt),
+            (
+                'user',
+                [{
+                    'type': 'image_url',
+                    'image_url': {
+                        'url': 'data:image/jpeg;base64,{image_data}'
+                    },
+                }],
+            ),
+        ])
         self.chain = self.prompt | self.model
         self.transform = PIL_to_base64
 
     def encode_image(self, images):
         captions = []
         for image in images:
-            response = self.chain.invoke({"image_data": image})
+            response = self.chain.invoke({'image_data': image})
             captions.append(response.content)
         return captions
 
 
 class CLIPModel(Embeddings):
+
     def __init__(
         self,
         model_name: str,
-        revision: str = "master",
-        hub="modelscope",
-        device="cpu",
+        revision: str = 'master',
+        hub=HubType.MODELSCOPE,
+        device='cpu',
     ):
         self.device = device
         self.model_name = model_name
         self.revision = revision
 
         # Download the model if it doesn't exist locally
-        if not os.path.exists(model_name) and hub == "modelscope":
+        if not os.path.exists(model_name) and hub == HubType.MODELSCOPE:
             model_name = download_model(self.model_name, self.revision)
 
         # Load the model and processor
@@ -85,9 +88,7 @@ class CLIPModel(Embeddings):
 
     def encode_text(self, batch_texts: List[str] | List[List[str]]):
         if isinstance(batch_texts[0], list):
-            batch_texts = [
-                text for _, texts in enumerate(batch_texts) for text in texts
-            ]
+            batch_texts = [text for _, texts in enumerate(batch_texts) for text in texts]
         # Ensure that the input texts are within the token limit
         max_length = self.tokenizer.model_max_length
         if not max_length or max_length > 0xFFFFFF:
@@ -97,7 +98,7 @@ class CLIPModel(Embeddings):
             max_length=max_length,
             padding=True,
             truncation=True,
-            return_tensors="pt",
+            return_tensors='pt',
         )
 
         inputs = {k: v.to(self.device) for k, v in encoded_inputs.items()}
@@ -108,7 +109,7 @@ class CLIPModel(Embeddings):
         return text_features
 
     def encode_image(self, image):
-        batch_images = torch.stack([d["pixel_values"][0] for d in image])
+        batch_images = torch.stack([d['pixel_values'][0] for d in image])
         batch_images = batch_images.to(self.device)
         with torch.no_grad():
             image_features = self.model.get_image_features(batch_images)
@@ -126,24 +127,19 @@ class CLIPModel(Embeddings):
     def embed_image(self, uris: List[str]):
         # read image and transform
         images = [Image.open(image_path) for image_path in uris]
-        transformed_images = [
-            self.transform(
-                image,
-                return_tensors="pt",
-            )
-            for image in images
-        ]
+        transformed_images = [self.transform(
+            image,
+            return_tensors='pt',
+        ) for image in images]
         image_features = self.encode_image(transformed_images)
         return image_features.cpu().numpy().tolist()
 
 
-if __name__ == "__main__":
-    model = CLIPModel("AI-ModelScope/chinese-clip-vit-large-patch14-336px")
-    model.embed_image(
-        [
-            "custom_eval/multimodal/images/AMNH.jpg",
-            "custom_eval/multimodal/images/AMNH.jpg",
-        ]
-    )
-    model.encode_text(["我喜欢吃饭" * 1000])
-    print("done")
+if __name__ == '__main__':
+    model = CLIPModel('AI-ModelScope/chinese-clip-vit-large-patch14-336px')
+    model.embed_image([
+        'custom_eval/multimodal/images/AMNH.jpg',
+        'custom_eval/multimodal/images/AMNH.jpg',
+    ])
+    model.encode_text(['我喜欢吃饭' * 1000])
+    print('done')

evalscope/backend/rag_eval/utils/embedding.py

@@ -1,18 +1,21 @@
 import os
 import torch
-from typing import List, Optional, Union, Dict
+from langchain_core.embeddings import Embeddings
 from sentence_transformers import models
-from sentence_transformers.SentenceTransformer import SentenceTransformer
 from sentence_transformers.cross_encoder import CrossEncoder
+from sentence_transformers.SentenceTransformer import SentenceTransformer
 from torch import Tensor
+from typing import Dict, List, Optional, Union
+
 from evalscope.backend.rag_eval.utils.tools import download_model
+from evalscope.constants import HubType
 from evalscope.utils.logger import get_logger
-from langchain_core.embeddings import Embeddings
 
 logger = get_logger()
 
 
 class BaseModel(Embeddings):
+
     def __init__(
         self,
         model_name_or_path: str,
@@ -83,9 +86,8 @@ class BaseModel(Embeddings):
 
 
 class SentenceTransformerModel(BaseModel):
-    def __init__(
-        self, model_name_or_path: str, pooling_mode: Optional[str] = None, **kwargs
-    ):
+
+    def __init__(self, model_name_or_path: str, pooling_mode: Optional[str] = None, **kwargs):
         super().__init__(model_name_or_path, **kwargs)
 
         if not pooling_mode:
@@ -104,9 +106,7 @@ class SentenceTransformerModel(BaseModel):
                 word_embedding_model.get_word_embedding_dimension(),
                 pooling_mode=pooling_mode,
             )
-            self.model = SentenceTransformer(
-                modules=[word_embedding_model, pooling_model],
-            )
+            self.model = SentenceTransformer(modules=[word_embedding_model, pooling_model], )
 
         self.model.max_seq_length = self.max_seq_length
 
@@ -130,6 +130,7 @@ class SentenceTransformerModel(BaseModel):
 
 
 class CrossEncoderModel(BaseModel):
+
     def __init__(self, model_name_or_path: str, **kwargs):
         super().__init__(model_name_or_path, **kwargs)
         self.model = CrossEncoder(
@@ -160,12 +161,12 @@ class EmbeddingModel:
     def load(
         model_name_or_path: str = '',
         is_cross_encoder: bool = False,
-        hub: str = 'modelscope',
+        hub: str = HubType.MODELSCOPE,
         revision: Optional[str] = 'master',
         **kwargs,
     ):
         # If model path does not exist and hub is 'modelscope', download the model
-        if not os.path.exists(model_name_or_path) and hub == 'modelscope':
+        if not os.path.exists(model_name_or_path) and hub == HubType.MODELSCOPE:
             model_name_or_path = download_model(model_name_or_path, revision)
 
         # Return different model instances based on whether it is a cross-encoder and pooling mode

evalscope/backend/rag_eval/utils/llm.py

@@ -1,13 +1,16 @@
 import os
-from typing import Any, Dict, Iterator, List, Mapping, Optional
-from modelscope.utils.hf_util import GenerationConfig
 from langchain_core.callbacks.manager import CallbackManagerForLLMRun
 from langchain_core.language_models.llms import LLM as BaseLLM
-from evalscope.models.model_adapter import ChatGenerationModelAdapter
 from langchain_openai import ChatOpenAI
+from modelscope.utils.hf_util import GenerationConfig
+from typing import Any, Dict, Iterator, List, Mapping, Optional
+
+from evalscope.constants import DEFAULT_MODEL_REVISION
+from evalscope.models.model_adapter import ChatGenerationModelAdapter
 
 
 class LLM:
+
     @staticmethod
     def load(**kw):
         api_base = kw.get('api_base', None)
@@ -25,8 +28,8 @@ class LocalLLM(BaseLLM):
     """A custom LLM that loads a model from a given path and performs inference."""
 
     model_name_or_path: str
-    model_revision: str = 'master'
-    template_type: str = 'default'
+    model_revision: str = DEFAULT_MODEL_REVISION
+    template_type: Optional[str] = None
     model_name: Optional[str]
     model: Optional[ChatGenerationModelAdapter]
     generation_config: Optional[Dict]
@@ -37,7 +40,6 @@ class LocalLLM(BaseLLM):
         self.model = ChatGenerationModelAdapter(
             model_id=self.model_name_or_path,
             model_revision=self.model_revision,
-            template_type=self.template_type,
             generation_config=GenerationConfig(**self.generation_config) if self.generation_config else None,
         )
 

evalscope/backend/rag_eval/utils/tools.py

@@ -1,7 +1,8 @@
+import base64
 import io
 import os
-import base64
 from modelscope import snapshot_download
+
 from evalscope.utils.logger import get_logger
 
 logger = get_logger()
@@ -9,9 +10,9 @@ logger = get_logger()
 
 def PIL_to_bytes(image_format, **kwargs):
     OPTIONS = {
-        "webp": dict(format="webp", lossless=True),
-        "png": dict(format="png"),
-        "jpg": dict(format="jpeg"),
+        'webp': dict(format='webp', lossless=True),
+        'png': dict(format='png'),
+        'jpg': dict(format='jpeg'),
     }
 
     def transform(image):
@@ -24,18 +25,18 @@ def PIL_to_bytes(image_format, **kwargs):
 
 def PIL_to_base64(image, **kwargs):
     bytestream = io.BytesIO()
-    image.save(bytestream, format="jpeg")
-    return base64.b64encode(bytestream.getvalue()).decode("utf-8")
+    image.save(bytestream, format='jpeg')
+    return base64.b64encode(bytestream.getvalue()).decode('utf-8')
 
 
 def path_to_bytes(filepath):
-    with open(filepath, "rb") as fp:
+    with open(filepath, 'rb') as fp:
         return fp.read()
 
 
 def path_to_base64(filepath):
     file_content = path_to_bytes(filepath)
-    return base64.b64encode(file_content).decode("utf-8")
+    return base64.b64encode(file_content).decode('utf-8')
 
 
 def ensure_dir(file_path):
@@ -44,19 +45,19 @@ def ensure_dir(file_path):
 
 def save_to_jsonl(df, file_path):
     ensure_dir(file_path)
-    df.to_json(file_path, orient="records", lines=True, force_ascii=False)
+    df.to_json(file_path, orient='records', lines=True, force_ascii=False)
 
 
 def save_to_tsv(df, file_path):
     ensure_dir(file_path)
-    df.to_csv(file_path, sep="\t", index=False)
+    df.to_csv(file_path, sep='\t', index=False)
 
 
 def download_model(model_id: str, revision: str):
     """
     default base dir: '~/.cache/modelscope/hub/model_id'
     """
-    logger.info(f"Loading model {model_id} from modelscope")
+    logger.info(f'Loading model {model_id} from modelscope')
 
     model_path = snapshot_download(model_id=model_id, revision=revision)
 

evalscope/backend/vlm_eval_kit/__init__.py

@@ -1 +1 @@
-from evalscope.backend.vlm_eval_kit.backend_manager import VLMEvalKitBackendManager
+from evalscope.backend.vlm_eval_kit.backend_manager import VLMEvalKitBackendManager

evalscope/backend/vlm_eval_kit/custom_dataset.py

@@ -1,32 +1,31 @@
-import os
 import numpy as np
+import os
 from vlmeval.dataset.image_base import ImageBaseDataset
 from vlmeval.dataset.image_vqa import CustomVQADataset
-from vlmeval.smp import load, dump, d2df
+from vlmeval.smp import d2df, dump, load
+
 
 class CustomDataset:
 
     def load_data(self, dataset):
         # customize the loading of the dataset
-        data_path = os.path.join(os.path.expanduser("~/LMUData"), f'{dataset}.tsv')
+        data_path = os.path.join(os.path.expanduser('~/LMUData'), f'{dataset}.tsv')
         return load(data_path)
 
-
     def build_prompt(self, line):
         msgs = ImageBaseDataset.build_prompt(self, line)
         # add a hint or custom instruction here
         msgs[-1]['value'] += '\nAnswer the question using a single word or phrase.'
         return msgs
-    
-    
+
     def evaluate(self, eval_file, **judge_kwargs):
         data = load(eval_file)
         assert 'answer' in data and 'prediction' in data
         data['prediction'] = [str(x) for x in data['prediction']]
         data['answer'] = [str(x).lower() for x in data['answer']]
-        
+
         print(data)
-        
+
         # ========compute the evaluation metrics as you need =========
         # exact match
         result = np.mean(data['answer'] == data['prediction'])

evalscope/benchmarks/arc/__init__.py

@@ -1,5 +1,6 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 
-from evalscope.benchmarks.arc.arc_adapter import ARCAdapter, DATASET_ID, SUBSET_LIST
+from evalscope.benchmarks.arc.arc_adapter import DATASET_ID, SUBSET_LIST
+from evalscope.benchmarks.arc.arc_adapter import ARCAdapter
 from evalscope.benchmarks.arc.arc_adapter import ARCAdapter as DataAdapterClass
-from evalscope.models.model_adapter import MultiChoiceModelAdapter as ModelAdapterClass    # noqa
+from evalscope.models.model_adapter import MultiChoiceModelAdapter as ModelAdapterClass  # noqa