evalscope 0.14.0__py3-none-any.whl → 0.15.1__py3-none-any.whl

evalscope/metrics/t2v_metrics/models/vqascore_models/mm_utils.py

@@ -0,0 +1,127 @@
+import os
+import torch
+from modelscope import AutoTokenizer
+from PIL import Image
+
+from ...constants import CACHE_DIR, IMAGE_TOKEN_INDEX
+
+
+def expand2square(pil_img, background_color):
+    width, height = pil_img.size
+    if width == height:
+        return pil_img
+    elif width > height:
+        result = Image.new(pil_img.mode, (width, width), background_color)
+        result.paste(pil_img, (0, (width - height) // 2))
+        return result
+    else:
+        result = Image.new(pil_img.mode, (height, height), background_color)
+        result.paste(pil_img, ((height - width) // 2, 0))
+        return result
+
+
+def tokenizer_image_token(prompt, tokenizer, image_token_index=IMAGE_TOKEN_INDEX, return_tensors=None):
+    prompt_chunks = [tokenizer(chunk).input_ids for chunk in prompt.split('<image>')]
+
+    def insert_separator(X, sep):
+        return [ele for sublist in zip(X, [sep] * len(X)) for ele in sublist][:-1]
+
+    input_ids = []
+    offset = 0
+    if len(prompt_chunks) > 0 and len(prompt_chunks[0]) > 0 and prompt_chunks[0][0] == tokenizer.bos_token_id:
+        offset = 1
+        input_ids.append(prompt_chunks[0][0])
+
+    for x in insert_separator(prompt_chunks, [image_token_index] * (offset + 1)):
+        input_ids.extend(x[offset:])
+
+    if return_tensors is not None:
+        if return_tensors == 'pt':
+            return torch.tensor(input_ids, dtype=torch.long)
+        raise ValueError(f'Unsupported tensor type: {return_tensors}')
+    return input_ids
+
+
+def t5_tokenizer_image_token(prompt, tokenizer, image_token_index=IMAGE_TOKEN_INDEX, return_tensors=None):
+    prompt_chunks = [tokenizer(chunk).input_ids for chunk in prompt.split('<image>')]
+
+    def insert_separator(X, sep):
+        return [ele for sublist in zip(X, [sep] * len(X)) for ele in sublist][:-1]
+
+    input_ids = []
+    # Since there's no bos_token_id, simply concatenate the tokenized prompt_chunks with the image_token_index
+    for x in insert_separator(prompt_chunks, [image_token_index]):
+        input_ids.extend(x)
+
+    if return_tensors is not None:
+        if return_tensors == 'pt':
+            return torch.tensor(input_ids, dtype=torch.long)
+        raise ValueError(f'Unsupported tensor type: {return_tensors}')
+    return input_ids
+
+
+def load_pretrained_model(
+        model_cls,
+        model_args,
+        model_path=None,
+        tokenizer_path=None,
+        model_max_length=None,
+        padding_side=None,
+        image_aspect_ratio='pad',  # or 'square'
+        mmprojector_repo=None,
+        mmprojector_name=None,
+        device='cuda',
+        cache_dir=CACHE_DIR):
+    tokenizer_dict = {}
+    if model_max_length:
+        tokenizer_dict['model_max_length'] = model_max_length
+    if padding_side:
+        tokenizer_dict['padding_side'] = padding_side
+
+    from ..utils import download_file
+
+    tokenizer = AutoTokenizer.from_pretrained(tokenizer_path, use_fast=False, **tokenizer_dict)
+    # tokenizer.pad_token = tokenizer.unk_token # could be redundant
+
+    model_path = download_file(model_path, cache_dir=cache_dir)
+    model = model_cls.from_pretrained(model_path, cache_dir=cache_dir)
+
+    if mmprojector_repo:
+        from huggingface_hub import hf_hub_download
+        model_base_name = mmprojector_repo.split('/')[-1]
+
+        if cache_dir is not None:
+            local_dir = os.path.join(cache_dir, model_base_name)
+        elif os.environ.get('HF_HOME') is not None:
+            local_dir = os.path.join(os.environ.get('HF_HOME'), model_base_name)
+        else:
+            local_dir = os.path.join(os.path.expanduser('~'), model_base_name)
+        print(f'Downloading projector weights to {local_dir}')
+        hf_hub_download(
+            repo_id=mmprojector_repo,
+            filename=mmprojector_name,
+            local_dir=local_dir,
+        )
+        pretrain_mm_mlp_adapter = os.path.join(local_dir, mmprojector_name)
+        model_args.pretrain_mm_mlp_adapter = pretrain_mm_mlp_adapter  # important to set to correct path
+
+        model.get_model().initialize_vision_modules(
+            model_args)  # This will load the CLIP vision encoder and MLP projector
+    else:
+        model.resize_token_embeddings(len(tokenizer))  # perhaps not needed
+
+    if not model.get_vision_tower().is_loaded:
+        model.get_vision_tower().load_model()
+    model.to(device=device, dtype=torch.bfloat16)
+    image_processor = model.get_vision_tower().image_processor
+
+    model.requires_grad_(False)
+
+    # below might be redundant
+    model.config.image_aspect_ratio = image_aspect_ratio
+    model.config.use_cache = False
+    model.config.image_grid_pinpoints = None
+    model.config.freeze_mm_mlp_adapter = True
+
+    model = model.eval()
+    return tokenizer, model, image_processor

evalscope/metrics/t2v_metrics/models/vqascore_models/vqa_model.py

@@ -0,0 +1,17 @@
+import torch
+from abc import abstractmethod
+from typing import List
+
+from ..model import ScoreModel
+
+
+class VQAScoreModel(ScoreModel):
+
+    @abstractmethod
+    def forward(self, images: List[str], texts: List[str], question_template: str,
+                answer_template: str) -> torch.Tensor:
+        """Forward pass of the model to return n scores for n (image, text) pairs (in PyTorch Tensor)
+        question_template: a string with optional {} to be replaced with the 'text'
+        answer_template: a string with optional {} to be replaced with the 'text'
+        """
+        pass

evalscope/metrics/t2v_metrics/score.py

@@ -0,0 +1,78 @@
+import torch
+import torch.nn as nn
+from abc import abstractmethod
+from torch.utils.data import DataLoader
+from tqdm import tqdm
+from typing import List, TypedDict, Union
+
+from .constants import CACHE_DIR
+
+
+class ImageTextDict(TypedDict):
+    images: List[str]
+    texts: List[str]
+
+
+class Score(nn.Module):
+
+    def __init__(self, model: str, device: str = 'cuda', cache_dir: str = CACHE_DIR, **kwargs):
+        """Initialize the ScoreModel
+        """
+        super().__init__()
+        assert model in self.list_all_models()
+        self.device = device
+        self.model = self.prepare_scoremodel(model, device, cache_dir, **kwargs)
+
+    @abstractmethod
+    def prepare_scoremodel(self, model: str, device: str, cache_dir: str, **kwargs):
+        """Prepare the ScoreModel
+        """
+        pass
+
+    @abstractmethod
+    def list_all_models(self) -> List[str]:
+        """List all available models
+        """
+        pass
+
+    def forward(self, images: Union[str, List[str]], texts: Union[str, List[str]], **kwargs) -> List[float]:
+        """Return the similarity score(s) between the image(s) and the text(s)
+        If there are m images and n texts, return a m x n tensor
+        """
+        if type(images) == str:
+            images = [images]
+        if type(texts) == str:
+            texts = [texts]
+        assert len(images) == len(texts), 'Number of images and texts must match'
+        scores = []
+        for i, image in enumerate(images):
+            scores.append(self.model.forward([image] * len(texts), texts, **kwargs))
+        return scores
+
+    def batch_forward(self, dataset: List[ImageTextDict], batch_size: int = 16, **kwargs) -> torch.Tensor:
+        """Return the similarity score(s) between the image(s) and the text(s)
+        If there are m images and n texts, return a m x n tensor
+        """
+        num_samples = len(dataset)
+        num_images = len(dataset[0]['images'])
+        num_texts = len(dataset[0]['texts'])
+        scores = torch.zeros(num_samples, num_images, num_texts).to(self.device)
+
+        dataloader = DataLoader(dataset, batch_size=batch_size, shuffle=False)
+        counter = 0
+        for batch_idx, batch in tqdm(enumerate(dataloader), total=len(dataloader)):
+            cur_batch_size = len(batch['images'][0])
+            assert len(batch['images']) == num_images, \
+                f"Number of image options in batch {batch_idx} is {len(batch['images'])}. Expected {num_images} images."
+            assert len(batch['texts']) == num_texts, \
+                f"Number of text options in batch {batch_idx} is {len(batch['texts'])}. Expected {num_texts} texts."
+
+            for image_idx in range(num_images):
+                images = batch['images'][image_idx]
+                for text_idx in range(num_texts):
+                    texts = batch['texts'][text_idx]
+                    scores[counter:counter+cur_batch_size, image_idx, text_idx] = \
+                        self.model.forward(images, texts, **kwargs)
+
+            counter += cur_batch_size
+        return scores

evalscope/metrics/t2v_metrics/vqascore.py

@@ -0,0 +1,14 @@
+from typing import List
+
+from .constants import CACHE_DIR
+from .models.vqascore_models import get_vqascore_model, list_all_vqascore_models
+from .score import Score
+
+
+class VQAScore(Score):
+
+    def prepare_scoremodel(self, model='clip-flant5-xxl', device='cuda', cache_dir=CACHE_DIR, **kwargs):
+        return get_vqascore_model(model, device=device, cache_dir=cache_dir, **kwargs)
+
+    def list_all_models(self) -> List[str]:
+        return list_all_vqascore_models()

evalscope/models/__init__.py

@@ -1,17 +1,53 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
+from typing import TYPE_CHECKING
 
-from evalscope.models.base_adapter import BaseModelAdapter, initialize_model_adapter
-from evalscope.models.chat_adapter import ChatGenerationModelAdapter
-from evalscope.models.choice_adapter import ContinuationLogitsModelAdapter, MultiChoiceModelAdapter
-from evalscope.models.custom import CustomModel
-from evalscope.models.custom_adapter import CustomModelAdapter
-from evalscope.models.local_model import LocalModel, get_local_model
-from evalscope.models.model import BaseModel, ChatBaseModel, OpenAIModel
-from evalscope.models.register import get_model_adapter
-from evalscope.models.server_adapter import ServerModelAdapter
+from evalscope.utils.import_utils import _LazyModule
 
-__all__ = [
-    'CustomModel', 'BaseModel', 'ChatBaseModel', 'OpenAIModel', 'BaseModelAdapter', 'ChatGenerationModelAdapter',
-    'MultiChoiceModelAdapter', 'ContinuationLogitsModelAdapter', 'CustomModelAdapter', 'ServerModelAdapter',
-    'LocalModel', 'get_local_model', 'initialize_model_adapter', 'get_model_adapter'
-]
+if TYPE_CHECKING:
+    from .adapters import (BaseModelAdapter, ChatGenerationModelAdapter, ContinuationLogitsModelAdapter,
+                           CustomModelAdapter, MultiChoiceModelAdapter, ServerModelAdapter, T2IModelAdapter,
+                           initialize_model_adapter)
+    from .custom import CustomModel, DummyCustomModel
+    from .local_model import LocalModel, get_local_model
+    from .model import BaseModel, ChatBaseModel, OpenAIModel
+    from .register import get_model_adapter
+
+else:
+    _import_structure = {
+        'adapters': [
+            'BaseModelAdapter',
+            'initialize_model_adapter',
+            'ChatGenerationModelAdapter',
+            'ContinuationLogitsModelAdapter',
+            'MultiChoiceModelAdapter',
+            'CustomModelAdapter',
+            'ServerModelAdapter',
+            'T2IModelAdapter',
+        ],
+        'custom': [
+            'CustomModel',
+            'DummyCustomModel',
+        ],
+        'local_model': [
+            'LocalModel',
+            'get_local_model',
+        ],
+        'model': [
+            'BaseModel',
+            'ChatBaseModel',
+            'OpenAIModel',
+        ],
+        'register': [
+            'get_model_adapter',
+        ],
+    }
+
+    import sys
+
+    sys.modules[__name__] = _LazyModule(
+        __name__,
+        globals()['__file__'],
+        _import_structure,
+        module_spec=__spec__,
+        extra_objects={},
+    )

evalscope/models/adapters/__init__.py

@@ -0,0 +1,17 @@
+from .base_adapter import BaseModelAdapter, initialize_model_adapter
+from .chat_adapter import ChatGenerationModelAdapter
+from .choice_adapter import ContinuationLogitsModelAdapter, MultiChoiceModelAdapter
+from .custom_adapter import CustomModelAdapter
+from .server_adapter import ServerModelAdapter
+from .t2i_adapter import T2IModelAdapter
+
+__all__ = [
+    'initialize_model_adapter',
+    'BaseModelAdapter',
+    'ChatGenerationModelAdapter',
+    'ContinuationLogitsModelAdapter',
+    'MultiChoiceModelAdapter',
+    'CustomModelAdapter',
+    'ServerModelAdapter',
+    'T2IModelAdapter',
+]

evalscope/models/{base_adapter.py → adapters/base_adapter.py}

@@ -3,19 +3,17 @@ from abc import ABC, abstractmethod
 from typing import TYPE_CHECKING, Any, List, Optional, Union
 
 from evalscope.constants import EvalType, OutputType
-from evalscope.models.custom import CustomModel
-from evalscope.models.local_model import LocalModel
-from evalscope.models.register import get_model_adapter, register_model_adapter
 from evalscope.utils.logger import get_logger
+from ..custom import CustomModel
+from ..local_model import LocalModel
 
 logger = get_logger()
 
 if TYPE_CHECKING:
-    from evalscope.benchmarks import BenchmarkMeta
+    from evalscope.benchmarks import DataAdapter
     from evalscope.config import TaskConfig
 
 
-@register_model_adapter('base')
 class BaseModelAdapter(ABC):
 
     def __init__(self, model: Optional[Union[LocalModel, CustomModel]], **kwargs):
@@ -39,12 +37,9 @@ class BaseModelAdapter(ABC):
         raise NotImplementedError
 
 
-def initialize_model_adapter(task_cfg: 'TaskConfig', benchmark: 'BenchmarkMeta', base_model: 'LocalModel'):
+def initialize_model_adapter(task_cfg: 'TaskConfig', benchmark: 'DataAdapter', base_model: 'LocalModel'):
     """Initialize the model adapter based on the task configuration."""
-    if task_cfg.dry_run:
-        from evalscope.models.model import DummyChatModel
-        return DummyChatModel(model_cfg=dict())
-    elif task_cfg.eval_type == EvalType.CUSTOM:
+    if task_cfg.eval_type == EvalType.CUSTOM:
         if not isinstance(task_cfg.model, CustomModel):
             raise ValueError(f'Expected evalscope.models.custom.CustomModel, but got {type(task_cfg.model)}.')
         from evalscope.models import CustomModelAdapter
@@ -66,13 +61,18 @@ def initialize_model_adapter(task_cfg: 'TaskConfig', benchmark: 'BenchmarkMeta',
             stream=task_cfg.stream,
         )
     else:
+        from ..register import get_model_adapter
+
         # for local model, we need to determine the model adapter class based on the output type
-        model_adapter_cls = benchmark.model_adapter
-        if model_adapter_cls not in benchmark.output_types:
-            logger.warning(f'Output type {model_adapter_cls} is not supported for benchmark {benchmark.name}. '
+        model_adapter_cls_str = benchmark.model_adapter
+        if model_adapter_cls_str not in benchmark.output_types:
+            logger.warning(f'Output type {model_adapter_cls_str} is not supported for benchmark {benchmark.name}. '
                            f'Using {benchmark.output_types[0]} instead.')
-            model_adapter_cls = benchmark.output_types[0]
+            model_adapter_cls_str = benchmark.output_types[0]
 
-        model_adapter = get_model_adapter(model_adapter_cls)
-        return model_adapter(
-            model=base_model, generation_config=task_cfg.generation_config, chat_template=task_cfg.chat_template)
+        model_adapter_cls = get_model_adapter(model_adapter_cls_str)
+        return model_adapter_cls(
+            model=base_model,
+            generation_config=task_cfg.generation_config,
+            chat_template=task_cfg.chat_template,
+            task_cfg=task_cfg)

evalscope/models/{chat_adapter.py → adapters/chat_adapter.py}

@@ -3,18 +3,15 @@ import time
 import torch
 from typing import Any, Dict, List, Tuple, Union
 
-from evalscope.constants import OutputType
-from evalscope.models.base_adapter import BaseModelAdapter
-from evalscope.models.local_model import LocalModel
-from evalscope.models.register import register_model_adapter
 from evalscope.utils.chat_service import ChatCompletionResponse, ChatCompletionResponseChoice, ChatMessage, Usage
 from evalscope.utils.logger import get_logger
 from evalscope.utils.model_utils import fix_do_sample_warning
+from ..local_model import LocalModel
+from .base_adapter import BaseModelAdapter
 
 logger = get_logger()
 
 
-@register_model_adapter(OutputType.GENERATION)
 class ChatGenerationModelAdapter(BaseModelAdapter):
     """
     Chat generation model adapter.
@@ -102,8 +99,14 @@ class ChatGenerationModelAdapter(BaseModelAdapter):
                 messages = [ChatMessage(role='user', content=query)]
                 if i < len(system_prompts) and system_prompts[i]:
                     messages = [ChatMessage(role='system', content=system_prompts[i])] + messages
-                formatted_prompts.append(
-                    self.tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True))
+                # whether thinking is needed
+                chat_template_kwargs = infer_cfg.get('chat_template_kwargs', None)
+                if chat_template_kwargs is not None:
+                    prompts = self.tokenizer.apply_chat_template(
+                        messages, tokenize=False, add_generation_prompt=True, **chat_template_kwargs)
+                else:
+                    prompts = self.tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
+                formatted_prompts.append(prompts)
         else:
             # For base model, use the queries as the input
             formatted_prompts = queries

evalscope/models/{choice_adapter.py → adapters/choice_adapter.py}

@@ -3,14 +3,11 @@ import time
 import torch
 from typing import List
 
-from evalscope.constants import OutputType
-from evalscope.models.base_adapter import BaseModelAdapter
-from evalscope.models.local_model import LocalModel
-from evalscope.models.register import register_model_adapter
 from evalscope.utils.chat_service import ChatCompletionResponse, ChatCompletionResponseChoice, ChatMessage
+from ..local_model import LocalModel
+from .base_adapter import BaseModelAdapter
 
 
-@register_model_adapter(OutputType.MULTIPLE_CHOICE)
 class MultiChoiceModelAdapter(BaseModelAdapter):
     """ The multi-choice model adapter. """
 
@@ -113,7 +110,6 @@ class MultiChoiceModelAdapter(BaseModelAdapter):
         return log_probs, {'tokens': tokens}
 
 
-@register_model_adapter(OutputType.CONTINUOUS)
 class ContinuationLogitsModelAdapter(MultiChoiceModelAdapter):
     """
     Continuation-logits model adapter.

evalscope/models/{custom_adapter.py → adapters/custom_adapter.py}

@@ -1,11 +1,9 @@
 from typing import Any, Dict, List, Union
 
-from evalscope.models.base_adapter import BaseModelAdapter
-from evalscope.models.custom import CustomModel
-from evalscope.models.register import register_model_adapter
+from ..custom import CustomModel
+from .base_adapter import BaseModelAdapter
 
 
-@register_model_adapter('custom')
 class CustomModelAdapter(BaseModelAdapter):
 
     def __init__(self, custom_model: CustomModel, **kwargs):

evalscope/models/{server_adapter.py → adapters/server_adapter.py}

@@ -5,14 +5,12 @@ from openai.types.chat import ChatCompletion, ChatCompletionChunk
 from openai.types.chat.chat_completion import ChatCompletionMessage, Choice
 from typing import List, Optional, Union
 
-from evalscope.models.base_adapter import BaseModelAdapter
-from evalscope.models.register import register_model_adapter
 from evalscope.utils.logger import get_logger
+from .base_adapter import BaseModelAdapter
 
 logger = get_logger()
 
 
-@register_model_adapter('server')
 class ServerModelAdapter(BaseModelAdapter):
     """
     Server model adapter to request remote API model and generate results.

evalscope/models/adapters/t2i_adapter.py

@@ -0,0 +1,76 @@
+import os
+import time
+import torch
+from typing import Any, Dict, List, Optional, Tuple, Union
+
+from evalscope.utils.chat_service import ChatCompletionResponse, ChatCompletionResponseChoice, ChatMessage
+from evalscope.utils.io_utils import OutputsStructure
+from evalscope.utils.logger import get_logger
+from ..local_model import LocalModel
+from .base_adapter import BaseModelAdapter
+
+logger = get_logger()
+
+
+class T2IModelAdapter(BaseModelAdapter):
+    """
+    Text to image model adapter.
+    """
+
+    def __init__(self, model: LocalModel, **kwargs):
+        super().__init__(model)
+
+        self.task_config = kwargs.get('task_cfg', None)
+        assert self.task_config is not None, 'Task config is required for T2I model adapter.'
+
+        self.save_path = os.path.join(self.task_config.work_dir, OutputsStructure.PREDICTIONS_DIR,
+                                      self.task_config.model_id, 'images')
+        os.makedirs(self.save_path, exist_ok=True)
+
+    def _model_generate(self, prompt, infer_cfg=None) -> List:
+        """
+        Generate images from the model.
+        Args:
+            prompt: The input prompt.
+            infer_cfg: The inference configuration.
+        Returns:
+            The generated images.
+        """
+        infer_cfg = infer_cfg or {}
+
+        sample = self.model(prompt=prompt, **infer_cfg).images
+        return sample
+
+    @torch.no_grad()
+    def predict(self, inputs: List[dict], infer_cfg: Optional[dict] = None) -> List[dict]:
+        """
+        Args:
+            inputs: The input data.
+            infer_cfg: The inference configuration.
+        Returns:
+            The prediction results.
+        """
+        results = []
+        for input_item in inputs:
+            prompt = input_item['data'][0]
+            image_id = input_item.get('id') or input_item.get('index')
+
+            samples = self._model_generate(prompt, infer_cfg)
+
+            choices_list = []
+            for index, sample in enumerate(samples):
+                image_file_path = os.path.join(self.save_path, f'{image_id}_{index}.jpeg')
+                sample.save(image_file_path)
+                logger.debug(f'Saved image to {image_file_path}')
+
+                choice = ChatCompletionResponseChoice(
+                    index=index, message=ChatMessage(content=image_file_path, role='assistant'), finish_reason='stop')
+                choices_list.append(choice)
+
+            res_d = ChatCompletionResponse(
+                model=self.model_id, choices=choices_list, object='images.generations',
+                created=int(time.time())).model_dump(exclude_unset=True)
+
+            results.append(res_d)
+
+        return results

evalscope/models/custom/__init__.py

@@ -1,3 +1,4 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 
-from evalscope.models.custom.custom_model import *
+from .custom_model import CustomModel
+from .dummy_model import DummyCustomModel

evalscope/models/custom/dummy_model.py

@@ -1,11 +1,10 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 import os
 import time
+from typing import List
 
-from evalscope.models.custom import CustomModel
-from evalscope.run import run_task
-from evalscope.utils.io_utils import yaml_to_dict
 from evalscope.utils.logger import get_logger
+from .custom_model import CustomModel
 
 logger = get_logger()
 """
@@ -15,29 +14,25 @@ This script is used to rewrite the evaluation results without re-running the mod
 
 class DummyCustomModel(CustomModel):
 
-    def __init__(self, config: dict, **kwargs):
+    def __init__(self, config: dict = {'model_id': 'dummy-model'}, **kwargs):
         super(DummyCustomModel, self).__init__(config=config, **kwargs)
 
-    def predict(self, prompts: str, **kwargs):
+    def predict(self, prompts: List[dict], **kwargs):
         # ONLY FOR DUMMY IMPLEMENTATION, DO NOT EDIT OR USE IN PRODUCTION.
 
-        response = 'The answer is C. NOTE: ONLY FOR TEST'
+        response = ''
 
         res_d: dict = {
             'choices': [{
                 'index': 0,
                 'message': {
-                    # 'content': f'The answer is B. Raw prompt: {prompt}',
                     'content': response,
                     'role': 'assistant'
                 }
             }],
-            'created':
-            time.time(),
-            'model':
-            self.config.get('model_id'),  # should be model_id
-            'object':
-            'chat.completion',
+            'created': time.time(),
+            'model': self.config.get('model_id'),  # should be model_id
+            'object': 'chat.completion',
             'usage': {
                 'completion_tokens': 0,
                 'prompt_tokens': 0,
@@ -49,6 +44,9 @@ class DummyCustomModel(CustomModel):
 
 
 if __name__ == '__main__':
+    from evalscope.run import run_task
+    from evalscope.utils.io_utils import yaml_to_dict
+
     # step1: 如果outputs做了迁移，需要修改outputs/eval_xxx 中的configs/task_output_config.yaml中的路径配置
     # step2: 执行此脚本，默认使用use_cache=True，实现免推理对eval结果进行刷新