evalscope 0.17.1__py3-none-any.whl → 1.0.1__py3-none-any.whl

evalscope/metrics/t2v_metrics/models/vqascore_models/mm_utils.py

@@ -61,17 +61,18 @@ def t5_tokenizer_image_token(prompt, tokenizer, image_token_index=IMAGE_TOKEN_IN
 
 
 def load_pretrained_model(
-        model_cls,
-        model_args,
-        model_path=None,
-        tokenizer_path=None,
-        model_max_length=None,
-        padding_side=None,
-        image_aspect_ratio='pad',  # or 'square'
-        mmprojector_repo=None,
-        mmprojector_name=None,
-        device='cuda',
-        cache_dir=CACHE_DIR):
+    model_cls,
+    model_args,
+    model_path=None,
+    tokenizer_path=None,
+    model_max_length=None,
+    padding_side=None,
+    image_aspect_ratio='pad',  # or 'square'
+    mmprojector_repo=None,
+    mmprojector_name=None,
+    device='cuda',
+    cache_dir=CACHE_DIR
+):
     tokenizer_dict = {}
     if model_max_length:
         tokenizer_dict['model_max_length'] = model_max_length
@@ -80,7 +81,7 @@ def load_pretrained_model(
 
     from ..utils import download_file
 
-    tokenizer = AutoTokenizer.from_pretrained(tokenizer_path, use_fast=False, **tokenizer_dict)
+    tokenizer = AutoTokenizer.from_pretrained(tokenizer_path, **tokenizer_dict)
     # tokenizer.pad_token = tokenizer.unk_token # could be redundant
 
     model_path = download_file(model_path, cache_dir=cache_dir)
@@ -106,7 +107,8 @@ def load_pretrained_model(
         model_args.pretrain_mm_mlp_adapter = pretrain_mm_mlp_adapter  # important to set to correct path
 
         model.get_model().initialize_vision_modules(
-            model_args)  # This will load the CLIP vision encoder and MLP projector
+            model_args
+        )  # This will load the CLIP vision encoder and MLP projector
     else:
         model.resize_token_embeddings(len(tokenizer))  # perhaps not needed
 

evalscope/metrics/t2v_metrics/models/vqascore_models/vqa_model.py

@@ -8,8 +8,9 @@ from ..model import ScoreModel
 class VQAScoreModel(ScoreModel):
 
     @abstractmethod
-    def forward(self, images: List[str], texts: List[str], question_template: str,
-                answer_template: str) -> torch.Tensor:
+    def forward(
+        self, images: List[str], texts: List[str], question_template: str, answer_template: str
+    ) -> torch.Tensor:
         """Forward pass of the model to return n scores for n (image, text) pairs (in PyTorch Tensor)
         question_template: a string with optional {} to be replaced with the 'text'
         answer_template: a string with optional {} to be replaced with the 'text'

evalscope/models/__init__.py

@@ -4,38 +4,15 @@ from typing import TYPE_CHECKING
 from evalscope.utils.import_utils import _LazyModule
 
 if TYPE_CHECKING:
-    from .adapters import (BaseModelAdapter, BFCLAdapter, ChatGenerationModelAdapter, ContinuationLogitsModelAdapter,
-                           CustomModelAdapter, MultiChoiceModelAdapter, ServerModelAdapter, T2IModelAdapter,
-                           TauBenchAdapter, initialize_model_adapter)
-    from .custom import CustomModel, DummyCustomModel
-    from .local_model import LocalModel, get_local_model
-    from .register import get_model_adapter
+    from .model_apis import llm_ckpt, mockllm, openai_api
 
 else:
     _import_structure = {
-        'adapters': [
-            'BaseModelAdapter',
-            'initialize_model_adapter',
-            'ChatGenerationModelAdapter',
-            'ContinuationLogitsModelAdapter',
-            'MultiChoiceModelAdapter',
-            'CustomModelAdapter',
-            'ServerModelAdapter',
-            'T2IModelAdapter',
-            'TauBenchAdapter',
-            'BFCLAdapter',
-        ],
-        'custom': [
-            'CustomModel',
-            'DummyCustomModel',
-        ],
-        'local_model': [
-            'LocalModel',
-            'get_local_model',
-        ],
-        'register': [
-            'get_model_adapter',
-        ],
+        'model_apis': [
+            'openai_api',
+            'mockllm',
+            'llm_ckpt',
+        ]
     }
 
     import sys

evalscope/models/image_edit_model.py

@@ -0,0 +1,125 @@
+from __future__ import annotations
+
+import importlib
+import time
+import torch
+from logging import getLogger
+from typing import Any, Dict, List, Literal, Optional, Protocol, Tuple, Union, cast
+
+from evalscope.api.messages import (
+    ChatMessage,
+    ChatMessageAssistant,
+    ContentAudio,
+    ContentImage,
+    ContentText,
+    ContentVideo,
+)
+from evalscope.api.model import (
+    ChatCompletionChoice,
+    GenerateConfig,
+    Logprob,
+    Logprobs,
+    ModelAPI,
+    ModelOutput,
+    ModelUsage,
+    TopLogprob,
+)
+from evalscope.api.tool import ToolChoice, ToolInfo
+from evalscope.utils.io_utils import PIL_to_base64, base64_to_PIL
+from evalscope.utils.model_utils import get_device
+
+logger = getLogger()
+
+
+class ImageEditAPI(ModelAPI):
+
+    def __init__(
+        self,
+        model_name: str,
+        base_url: Optional[str] = None,
+        api_key: Optional[str] = None,
+        config: GenerateConfig = GenerateConfig(),
+        **model_args: Any,
+    ):
+        super().__init__(
+            model_name=model_name,
+            base_url=base_url,
+            api_key=api_key,
+            config=config,
+        )
+
+        # collect known model_args (then delete them so we can pass the rest on)
+        def collect_model_arg(name: str) -> Optional[Any]:
+            nonlocal model_args
+            value = model_args.get(name, None)
+            if value is not None:
+                model_args.pop(name)
+            return value
+
+        model_path = collect_model_arg('model_path')
+        torch_dtype = collect_model_arg('precision') or collect_model_arg('torch_dtype')
+        device_map = collect_model_arg('device_map')
+        # torch dtype
+        DTYPE_MAP = {'float16': torch.float16, 'float32': torch.float32, 'bfloat16': torch.bfloat16, 'auto': 'auto'}
+
+        if isinstance(torch_dtype, str) and torch_dtype != 'auto':
+            torch_dtype = DTYPE_MAP.get(torch_dtype, torch.float32)
+        self.torch_dtype = torch_dtype
+        self.device = device_map or get_device()
+
+        self.pipeline_cls = collect_model_arg('pipeline_cls')
+        # default to DiffusionPipeline if not specified
+        if self.pipeline_cls is None:
+            if 'qwen' in model_name.lower():
+                self.pipeline_cls = 'QwenImageEditPipeline'
+            else:
+                logger.error('Pipeline class not found. Please provide a valid `pipeline_cls` in model args.')
+                raise ValueError('Invalid pipeline class.')
+
+        model_name_or_path = model_path or model_name
+
+        # from modelscope import pipeline_cls
+        module = getattr(importlib.import_module('modelscope'), self.pipeline_cls)
+        logger.info(f'Loading model {model_name_or_path} with {self.pipeline_cls} ...')
+
+        self.model = module.from_pretrained(
+            model_name_or_path,
+            torch_dtype=self.torch_dtype,
+            **model_args,
+        )
+
+        self.model.to(self.device)
+
+    def generate(
+        self,
+        input: List[ChatMessage],
+        tools: List[ToolInfo],
+        tool_choice: ToolChoice,
+        config: GenerateConfig,
+    ) -> ModelOutput:
+
+        # prepare generator
+        kwargs: Dict[str, Any] = {}
+        if config.num_inference_steps is not None:
+            kwargs['num_inference_steps'] = config.num_inference_steps
+        kwargs.update(config.model_extra)
+
+        # assume the first text as prompt
+        content = input[0].content
+        assert isinstance(content[0], ContentText) and isinstance(content[1], ContentImage), \
+            'Invalid content types, expected (ContentText, ContentImage)'
+
+        prompt = content[0].text
+        input_image_base64 = content[1].image
+        input_image = base64_to_PIL(input_image_base64)
+        # get the first image as output
+        output = self.model(image=input_image, prompt=prompt, **kwargs)
+        image = output.images[0]
+
+        image_base64 = PIL_to_base64(image)
+
+        return ModelOutput(
+            model=self.model_name,
+            choices=[ChatCompletionChoice.from_content(content=[ContentImage(image=image_base64)])],
+            time=time.time(),
+        )

evalscope/models/mockllm.py

@@ -0,0 +1,65 @@
+from typing import Any, Dict, Generator, Iterable, Iterator, List, Optional, Union
+
+from evalscope.api.dataset import Dataset
+from evalscope.api.messages import ChatMessage
+from evalscope.api.model import GenerateConfig, ModelAPI, ModelOutput
+from evalscope.api.tool import ToolChoice, ToolInfo
+from evalscope.utils.function_utils import thread_safe
+
+
+class MockLLM(ModelAPI):
+    """A mock implementation of the ModelAPI class for testing purposes.
+
+    Always returns default_output, unless you pass in a model_args
+    key "custom_outputs" with a value of an Iterable[ModelOutput]
+    """
+
+    default_output = 'Default output from mockllm/model'
+
+    outputs: Iterator[ModelOutput]
+
+    def __init__(
+        self,
+        model_name: str,
+        base_url: Optional[str] = None,
+        api_key: Optional[str] = None,
+        config: GenerateConfig = GenerateConfig(),
+        custom_outputs: Iterable[ModelOutput] = None,
+        **model_args: Dict[str, Any],
+    ) -> None:
+        super().__init__(model_name, base_url, api_key, config)
+        self.model_args = model_args
+        if custom_outputs is not None:
+            # We cannot rely on the user of this model giving custom_outputs
+            # the correct type since they do not call this constructor
+            # Hence this type check and the one in generate.
+            if not isinstance(custom_outputs, (Iterable, Generator)):
+                raise ValueError(
+                    f"model_args['custom_outputs'] must be an Iterable or a Generator, got {custom_outputs}"
+                )
+            self.outputs = iter(custom_outputs)
+        else:
+            self.outputs = iter((
+                ModelOutput.from_content(model='mockllm', content=self.default_output)
+                for _ in iter(int, 1)  # produce an infinite iterator
+            ))
+
+    @thread_safe
+    def generate(
+        self,
+        input: List[ChatMessage],
+        tools: List[ToolInfo],
+        tool_choice: ToolChoice,
+        config: GenerateConfig,
+    ) -> ModelOutput:
+        try:
+            output = next(self.outputs)
+        except StopIteration:
+            raise ValueError('custom_outputs ran out of values')
+
+        if not isinstance(output, ModelOutput):
+            raise ValueError(f'output must be an instance of ModelOutput; got {type(output)}; content: {repr(output)}')
+        return output
+
+    def batch_generate(inputs: Dataset, config: GenerateConfig) -> List[ModelOutput]:
+        return super().batch_generate(inputs, config)

evalscope/models/model_apis.py

@@ -0,0 +1,67 @@
+from evalscope.api.model import ModelAPI
+from evalscope.api.registry import register_model_api
+from evalscope.utils.deprecation_utils import deprecated
+from evalscope.utils.import_utils import check_import
+
+
+@register_model_api(name='mock_llm')
+def mockllm() -> type[ModelAPI]:
+    from .mockllm import MockLLM
+
+    return MockLLM
+
+
+@register_model_api(name='openai_api')
+def openai_api() -> type[ModelAPI]:
+    from .openai_compatible import OpenAICompatibleAPI
+
+    return OpenAICompatibleAPI
+
+
+@register_model_api(name='server')
+@deprecated(since='1.0.0', remove_in='1.1.0', alternative='openai_api')
+def server() -> type[ModelAPI]:
+    from .openai_compatible import OpenAICompatibleAPI
+
+    return OpenAICompatibleAPI
+
+
+@register_model_api(name='llm_ckpt')
+def llm_ckpt() -> type[ModelAPI]:
+    check_import('torch', package='torch', raise_error=True)
+
+    from .modelscope import ModelScopeAPI
+
+    return ModelScopeAPI
+
+
+@register_model_api(name='checkpoint')
+@deprecated(since='1.0.0', remove_in='1.1.0', alternative='llm_ckpt')
+def checkpoint() -> type[ModelAPI]:
+    check_import('torch', package='torch', raise_error=True)
+
+    from .modelscope import ModelScopeAPI
+
+    return ModelScopeAPI
+
+
+@register_model_api(name='text2image')
+def text2image() -> type[ModelAPI]:
+    check_import('torch', package='evalscope[aigc]', raise_error=True)
+    check_import('torchvision', package='evalscope[aigc]', raise_error=True)
+    check_import('diffusers', package='evalscope[aigc]', raise_error=True)
+
+    from .text2image_model import Text2ImageAPI
+
+    return Text2ImageAPI
+
+
+@register_model_api(name='image_editing')
+def image_editing() -> type[ModelAPI]:
+    check_import('torch', package='evalscope[aigc]', raise_error=True)
+    check_import('torchvision', package='evalscope[aigc]', raise_error=True)
+    check_import('diffusers', package='evalscope[aigc]', raise_error=True)
+
+    from .image_edit_model import ImageEditAPI
+
+    return ImageEditAPI