evalscope 0.10.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

evalscope/metrics/t2v_metrics/models/vqascore_models/mm_utils.py

@@ -0,0 +1,129 @@
+import os
+import torch
+from modelscope import AutoTokenizer
+from PIL import Image
+
+from ...constants import CACHE_DIR, IMAGE_TOKEN_INDEX
+
+
+def expand2square(pil_img, background_color):
+    width, height = pil_img.size
+    if width == height:
+        return pil_img
+    elif width > height:
+        result = Image.new(pil_img.mode, (width, width), background_color)
+        result.paste(pil_img, (0, (width - height) // 2))
+        return result
+    else:
+        result = Image.new(pil_img.mode, (height, height), background_color)
+        result.paste(pil_img, ((height - width) // 2, 0))
+        return result
+
+
+def tokenizer_image_token(prompt, tokenizer, image_token_index=IMAGE_TOKEN_INDEX, return_tensors=None):
+    prompt_chunks = [tokenizer(chunk).input_ids for chunk in prompt.split('<image>')]
+
+    def insert_separator(X, sep):
+        return [ele for sublist in zip(X, [sep] * len(X)) for ele in sublist][:-1]
+
+    input_ids = []
+    offset = 0
+    if len(prompt_chunks) > 0 and len(prompt_chunks[0]) > 0 and prompt_chunks[0][0] == tokenizer.bos_token_id:
+        offset = 1
+        input_ids.append(prompt_chunks[0][0])
+
+    for x in insert_separator(prompt_chunks, [image_token_index] * (offset + 1)):
+        input_ids.extend(x[offset:])
+
+    if return_tensors is not None:
+        if return_tensors == 'pt':
+            return torch.tensor(input_ids, dtype=torch.long)
+        raise ValueError(f'Unsupported tensor type: {return_tensors}')
+    return input_ids
+
+
+def t5_tokenizer_image_token(prompt, tokenizer, image_token_index=IMAGE_TOKEN_INDEX, return_tensors=None):
+    prompt_chunks = [tokenizer(chunk).input_ids for chunk in prompt.split('<image>')]
+
+    def insert_separator(X, sep):
+        return [ele for sublist in zip(X, [sep] * len(X)) for ele in sublist][:-1]
+
+    input_ids = []
+    # Since there's no bos_token_id, simply concatenate the tokenized prompt_chunks with the image_token_index
+    for x in insert_separator(prompt_chunks, [image_token_index]):
+        input_ids.extend(x)
+
+    if return_tensors is not None:
+        if return_tensors == 'pt':
+            return torch.tensor(input_ids, dtype=torch.long)
+        raise ValueError(f'Unsupported tensor type: {return_tensors}')
+    return input_ids
+
+
+def load_pretrained_model(
+    model_cls,
+    model_args,
+    model_path=None,
+    tokenizer_path=None,
+    model_max_length=None,
+    padding_side=None,
+    image_aspect_ratio='pad',  # or 'square'
+    mmprojector_repo=None,
+    mmprojector_name=None,
+    device='cuda',
+    cache_dir=CACHE_DIR
+):
+    tokenizer_dict = {}
+    if model_max_length:
+        tokenizer_dict['model_max_length'] = model_max_length
+    if padding_side:
+        tokenizer_dict['padding_side'] = padding_side
+
+    from ..utils import download_file
+
+    tokenizer = AutoTokenizer.from_pretrained(tokenizer_path, **tokenizer_dict)
+    # tokenizer.pad_token = tokenizer.unk_token # could be redundant
+
+    model_path = download_file(model_path, cache_dir=cache_dir)
+    model = model_cls.from_pretrained(model_path, cache_dir=cache_dir)
+
+    if mmprojector_repo:
+        from huggingface_hub import hf_hub_download
+        model_base_name = mmprojector_repo.split('/')[-1]
+
+        if cache_dir is not None:
+            local_dir = os.path.join(cache_dir, model_base_name)
+        elif os.environ.get('HF_HOME') is not None:
+            local_dir = os.path.join(os.environ.get('HF_HOME'), model_base_name)
+        else:
+            local_dir = os.path.join(os.path.expanduser('~'), model_base_name)
+        print(f'Downloading projector weights to {local_dir}')
+        hf_hub_download(
+            repo_id=mmprojector_repo,
+            filename=mmprojector_name,
+            local_dir=local_dir,
+        )
+        pretrain_mm_mlp_adapter = os.path.join(local_dir, mmprojector_name)
+        model_args.pretrain_mm_mlp_adapter = pretrain_mm_mlp_adapter  # important to set to correct path
+
+        model.get_model().initialize_vision_modules(
+            model_args
+        )  # This will load the CLIP vision encoder and MLP projector
+    else:
+        model.resize_token_embeddings(len(tokenizer))  # perhaps not needed
+
+    if not model.get_vision_tower().is_loaded:
+        model.get_vision_tower().load_model()
+    model.to(device=device, dtype=torch.bfloat16)
+    image_processor = model.get_vision_tower().image_processor
+
+    model.requires_grad_(False)
+
+    # below might be redundant
+    model.config.image_aspect_ratio = image_aspect_ratio
+    model.config.use_cache = False
+    model.config.image_grid_pinpoints = None
+    model.config.freeze_mm_mlp_adapter = True
+
+    model = model.eval()
+    return tokenizer, model, image_processor

evalscope/metrics/t2v_metrics/models/vqascore_models/vqa_model.py

@@ -0,0 +1,18 @@
+import torch
+from abc import abstractmethod
+from typing import List
+
+from ..model import ScoreModel
+
+
+class VQAScoreModel(ScoreModel):
+
+    @abstractmethod
+    def forward(
+        self, images: List[str], texts: List[str], question_template: str, answer_template: str
+    ) -> torch.Tensor:
+        """Forward pass of the model to return n scores for n (image, text) pairs (in PyTorch Tensor)
+        question_template: a string with optional {} to be replaced with the 'text'
+        answer_template: a string with optional {} to be replaced with the 'text'
+        """
+        pass

evalscope/metrics/t2v_metrics/score.py

@@ -0,0 +1,78 @@
+import torch
+import torch.nn as nn
+from abc import abstractmethod
+from torch.utils.data import DataLoader
+from tqdm import tqdm
+from typing import List, TypedDict, Union
+
+from .constants import CACHE_DIR
+
+
+class ImageTextDict(TypedDict):
+    images: List[str]
+    texts: List[str]
+
+
+class Score(nn.Module):
+
+    def __init__(self, model: str, device: str = 'cuda', cache_dir: str = CACHE_DIR, **kwargs):
+        """Initialize the ScoreModel
+        """
+        super().__init__()
+        assert model in self.list_all_models()
+        self.device = device
+        self.model = self.prepare_scoremodel(model, device, cache_dir, **kwargs)
+
+    @abstractmethod
+    def prepare_scoremodel(self, model: str, device: str, cache_dir: str, **kwargs):
+        """Prepare the ScoreModel
+        """
+        pass
+
+    @abstractmethod
+    def list_all_models(self) -> List[str]:
+        """List all available models
+        """
+        pass
+
+    def forward(self, images: Union[str, List[str]], texts: Union[str, List[str]], **kwargs) -> List[float]:
+        """Return the similarity score(s) between the image(s) and the text(s)
+        If there are m images and n texts, return a m x n tensor
+        """
+        if type(images) == str:
+            images = [images]
+        if type(texts) == str:
+            texts = [texts]
+        assert len(images) == len(texts), 'Number of images and texts must match'
+        scores = []
+        for i, image in enumerate(images):
+            scores.append(self.model.forward([image] * len(texts), texts, **kwargs))
+        return scores
+
+    def batch_forward(self, dataset: List[ImageTextDict], batch_size: int = 16, **kwargs) -> torch.Tensor:
+        """Return the similarity score(s) between the image(s) and the text(s)
+        If there are m images and n texts, return a m x n tensor
+        """
+        num_samples = len(dataset)
+        num_images = len(dataset[0]['images'])
+        num_texts = len(dataset[0]['texts'])
+        scores = torch.zeros(num_samples, num_images, num_texts).to(self.device)
+
+        dataloader = DataLoader(dataset, batch_size=batch_size, shuffle=False)
+        counter = 0
+        for batch_idx, batch in tqdm(enumerate(dataloader), total=len(dataloader)):
+            cur_batch_size = len(batch['images'][0])
+            assert len(batch['images']) == num_images, \
+                f"Number of image options in batch {batch_idx} is {len(batch['images'])}. Expected {num_images} images."
+            assert len(batch['texts']) == num_texts, \
+                f"Number of text options in batch {batch_idx} is {len(batch['texts'])}. Expected {num_texts} texts."
+
+            for image_idx in range(num_images):
+                images = batch['images'][image_idx]
+                for text_idx in range(num_texts):
+                    texts = batch['texts'][text_idx]
+                    scores[counter:counter+cur_batch_size, image_idx, text_idx] = \
+                        self.model.forward(images, texts, **kwargs)
+
+            counter += cur_batch_size
+        return scores

evalscope/metrics/t2v_metrics/vqascore.py

@@ -0,0 +1,14 @@
+from typing import List
+
+from .constants import CACHE_DIR
+from .models.vqascore_models import get_vqascore_model, list_all_vqascore_models
+from .score import Score
+
+
+class VQAScore(Score):
+
+    def prepare_scoremodel(self, model='clip-flant5-xxl', device='cuda', cache_dir=CACHE_DIR, **kwargs):
+        return get_vqascore_model(model, device=device, cache_dir=cache_dir, **kwargs)
+
+    def list_all_models(self) -> List[str]:
+        return list_all_vqascore_models()

evalscope/models/__init__.py

@@ -1,16 +1,26 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
+from typing import TYPE_CHECKING
 
-from evalscope.models.base_adapter import BaseModelAdapter, initialize_model_adapter
-from evalscope.models.chat_adapter import ChatGenerationModelAdapter
-from evalscope.models.choice_adapter import ContinuationLogitsModelAdapter, MultiChoiceModelAdapter
-from evalscope.models.custom import CustomModel
-from evalscope.models.custom_adapter import CustomModelAdapter
-from evalscope.models.local_model import LocalModel, get_local_model
-from evalscope.models.model import BaseModel, ChatBaseModel, OpenAIModel
-from evalscope.models.server_adapter import ServerModelAdapter
+from evalscope.utils.import_utils import _LazyModule
 
-__all__ = [
-    'CustomModel', 'BaseModel', 'ChatBaseModel', 'OpenAIModel', 'BaseModelAdapter', 'ChatGenerationModelAdapter',
-    'MultiChoiceModelAdapter', 'ContinuationLogitsModelAdapter', 'CustomModelAdapter', 'ServerModelAdapter',
-    'LocalModel', 'get_local_model', 'initialize_model_adapter'
-]
+if TYPE_CHECKING:
+    from .model_apis import llm_ckpt, mockllm, openai_api
+
+else:
+    _import_structure = {
+        'model_apis': [
+            'openai_api',
+            'mockllm',
+            'llm_ckpt',
+        ]
+    }
+
+    import sys
+
+    sys.modules[__name__] = _LazyModule(
+        __name__,
+        globals()['__file__'],
+        _import_structure,
+        module_spec=__spec__,
+        extra_objects={},
+    )

evalscope/models/image_edit_model.py

@@ -0,0 +1,125 @@
+from __future__ import annotations
+
+import importlib
+import time
+import torch
+from logging import getLogger
+from typing import Any, Dict, List, Literal, Optional, Protocol, Tuple, Union, cast
+
+from evalscope.api.messages import (
+    ChatMessage,
+    ChatMessageAssistant,
+    ContentAudio,
+    ContentImage,
+    ContentText,
+    ContentVideo,
+)
+from evalscope.api.model import (
+    ChatCompletionChoice,
+    GenerateConfig,
+    Logprob,
+    Logprobs,
+    ModelAPI,
+    ModelOutput,
+    ModelUsage,
+    TopLogprob,
+)
+from evalscope.api.tool import ToolChoice, ToolInfo
+from evalscope.utils.io_utils import PIL_to_base64, base64_to_PIL
+from evalscope.utils.model_utils import get_device
+
+logger = getLogger()
+
+
+class ImageEditAPI(ModelAPI):
+
+    def __init__(
+        self,
+        model_name: str,
+        base_url: Optional[str] = None,
+        api_key: Optional[str] = None,
+        config: GenerateConfig = GenerateConfig(),
+        **model_args: Any,
+    ):
+        super().__init__(
+            model_name=model_name,
+            base_url=base_url,
+            api_key=api_key,
+            config=config,
+        )
+
+        # collect known model_args (then delete them so we can pass the rest on)
+        def collect_model_arg(name: str) -> Optional[Any]:
+            nonlocal model_args
+            value = model_args.get(name, None)
+            if value is not None:
+                model_args.pop(name)
+            return value
+
+        model_path = collect_model_arg('model_path')
+        torch_dtype = collect_model_arg('precision') or collect_model_arg('torch_dtype')
+        device_map = collect_model_arg('device_map')
+        # torch dtype
+        DTYPE_MAP = {'float16': torch.float16, 'float32': torch.float32, 'bfloat16': torch.bfloat16, 'auto': 'auto'}
+
+        if isinstance(torch_dtype, str) and torch_dtype != 'auto':
+            torch_dtype = DTYPE_MAP.get(torch_dtype, torch.float32)
+        self.torch_dtype = torch_dtype
+        self.device = device_map or get_device()
+
+        self.pipeline_cls = collect_model_arg('pipeline_cls')
+        # default to DiffusionPipeline if not specified
+        if self.pipeline_cls is None:
+            if 'qwen' in model_name.lower():
+                self.pipeline_cls = 'QwenImageEditPipeline'
+            else:
+                logger.error('Pipeline class not found. Please provide a valid `pipeline_cls` in model args.')
+                raise ValueError('Invalid pipeline class.')
+
+        model_name_or_path = model_path or model_name
+
+        # from modelscope import pipeline_cls
+        module = getattr(importlib.import_module('modelscope'), self.pipeline_cls)
+        logger.info(f'Loading model {model_name_or_path} with {self.pipeline_cls} ...')
+
+        self.model = module.from_pretrained(
+            model_name_or_path,
+            torch_dtype=self.torch_dtype,
+            **model_args,
+        )
+
+        self.model.to(self.device)
+
+    def generate(
+        self,
+        input: List[ChatMessage],
+        tools: List[ToolInfo],
+        tool_choice: ToolChoice,
+        config: GenerateConfig,
+    ) -> ModelOutput:
+
+        # prepare generator
+        kwargs: Dict[str, Any] = {}
+        if config.num_inference_steps is not None:
+            kwargs['num_inference_steps'] = config.num_inference_steps
+        kwargs.update(config.model_extra)
+
+        # assume the first text as prompt
+        content = input[0].content
+        assert isinstance(content[0], ContentText) and isinstance(content[1], ContentImage), \
+            'Invalid content types, expected (ContentText, ContentImage)'
+
+        prompt = content[0].text
+        input_image_base64 = content[1].image
+        input_image = base64_to_PIL(input_image_base64)
+        # get the first image as output
+        output = self.model(image=input_image, prompt=prompt, **kwargs)
+        image = output.images[0]
+
+        image_base64 = PIL_to_base64(image)
+
+        return ModelOutput(
+            model=self.model_name,
+            choices=[ChatCompletionChoice.from_content(content=[ContentImage(image=image_base64)])],
+            time=time.time(),
+        )

evalscope/models/mockllm.py

@@ -0,0 +1,65 @@
+from typing import Any, Dict, Generator, Iterable, Iterator, List, Optional, Union
+
+from evalscope.api.dataset import Dataset
+from evalscope.api.messages import ChatMessage
+from evalscope.api.model import GenerateConfig, ModelAPI, ModelOutput
+from evalscope.api.tool import ToolChoice, ToolInfo
+from evalscope.utils.function_utils import thread_safe
+
+
+class MockLLM(ModelAPI):
+    """A mock implementation of the ModelAPI class for testing purposes.
+
+    Always returns default_output, unless you pass in a model_args
+    key "custom_outputs" with a value of an Iterable[ModelOutput]
+    """
+
+    default_output = 'Default output from mockllm/model'
+
+    outputs: Iterator[ModelOutput]
+
+    def __init__(
+        self,
+        model_name: str,
+        base_url: Optional[str] = None,
+        api_key: Optional[str] = None,
+        config: GenerateConfig = GenerateConfig(),
+        custom_outputs: Iterable[ModelOutput] = None,
+        **model_args: Dict[str, Any],
+    ) -> None:
+        super().__init__(model_name, base_url, api_key, config)
+        self.model_args = model_args
+        if custom_outputs is not None:
+            # We cannot rely on the user of this model giving custom_outputs
+            # the correct type since they do not call this constructor
+            # Hence this type check and the one in generate.
+            if not isinstance(custom_outputs, (Iterable, Generator)):
+                raise ValueError(
+                    f"model_args['custom_outputs'] must be an Iterable or a Generator, got {custom_outputs}"
+                )
+            self.outputs = iter(custom_outputs)
+        else:
+            self.outputs = iter((
+                ModelOutput.from_content(model='mockllm', content=self.default_output)
+                for _ in iter(int, 1)  # produce an infinite iterator
+            ))
+
+    @thread_safe
+    def generate(
+        self,
+        input: List[ChatMessage],
+        tools: List[ToolInfo],
+        tool_choice: ToolChoice,
+        config: GenerateConfig,
+    ) -> ModelOutput:
+        try:
+            output = next(self.outputs)
+        except StopIteration:
+            raise ValueError('custom_outputs ran out of values')
+
+        if not isinstance(output, ModelOutput):
+            raise ValueError(f'output must be an instance of ModelOutput; got {type(output)}; content: {repr(output)}')
+        return output
+
+    def batch_generate(inputs: Dataset, config: GenerateConfig) -> List[ModelOutput]:
+        return super().batch_generate(inputs, config)

evalscope/models/model_apis.py

@@ -0,0 +1,69 @@
+from evalscope.api.model import ModelAPI
+from evalscope.api.registry import register_model_api
+from evalscope.utils.deprecation_utils import deprecated
+from evalscope.utils.import_utils import check_import
+
+
+@register_model_api(name='mock_llm')
+def mockllm() -> type[ModelAPI]:
+    from .mockllm import MockLLM
+
+    return MockLLM
+
+
+@register_model_api(name='openai_api')
+def openai_api() -> type[ModelAPI]:
+    from .openai_compatible import OpenAICompatibleAPI
+
+    return OpenAICompatibleAPI
+
+
+@register_model_api(name='server')
+@deprecated(since='1.0.0', remove_in='1.1.0', alternative='openai_api')
+def server() -> type[ModelAPI]:
+    from .openai_compatible import OpenAICompatibleAPI
+
+    return OpenAICompatibleAPI
+
+
+@register_model_api(name='llm_ckpt')
+def llm_ckpt() -> type[ModelAPI]:
+    check_import('torch', package='torch', raise_error=True, feature_name='llm_ckpt')
+
+    from .modelscope import ModelScopeAPI
+
+    return ModelScopeAPI
+
+
+@register_model_api(name='checkpoint')
+@deprecated(since='1.0.0', remove_in='1.1.0', alternative='llm_ckpt')
+def checkpoint() -> type[ModelAPI]:
+    check_import('torch', package='torch', raise_error=True, feature_name='llm_ckpt')
+
+    from .modelscope import ModelScopeAPI
+
+    return ModelScopeAPI
+
+
+@register_model_api(name='text2image')
+def text2image() -> type[ModelAPI]:
+    check_import(['torch', 'torchvision', 'diffusers'],
+                 package='evalscope[aigc]',
+                 raise_error=True,
+                 feature_name='text2image')
+
+    from .text2image_model import Text2ImageAPI
+
+    return Text2ImageAPI
+
+
+@register_model_api(name='image_editing')
+def image_editing() -> type[ModelAPI]:
+    check_import(['torch', 'torchvision', 'diffusers'],
+                 package='evalscope[aigc]',
+                 raise_error=True,
+                 feature_name='image_editing')
+
+    from .image_edit_model import ImageEditAPI
+
+    return ImageEditAPI