evalscope 0.17.1__py3-none-any.whl → 1.0.0__py3-none-any.whl

evalscope/metrics/t2v_metrics/models/vqascore_models/mm_utils.py

@@ -61,17 +61,18 @@ def t5_tokenizer_image_token(prompt, tokenizer, image_token_index=IMAGE_TOKEN_IN
 
 
 def load_pretrained_model(
-        model_cls,
-        model_args,
-        model_path=None,
-        tokenizer_path=None,
-        model_max_length=None,
-        padding_side=None,
-        image_aspect_ratio='pad',  # or 'square'
-        mmprojector_repo=None,
-        mmprojector_name=None,
-        device='cuda',
-        cache_dir=CACHE_DIR):
+    model_cls,
+    model_args,
+    model_path=None,
+    tokenizer_path=None,
+    model_max_length=None,
+    padding_side=None,
+    image_aspect_ratio='pad',  # or 'square'
+    mmprojector_repo=None,
+    mmprojector_name=None,
+    device='cuda',
+    cache_dir=CACHE_DIR
+):
     tokenizer_dict = {}
     if model_max_length:
         tokenizer_dict['model_max_length'] = model_max_length
@@ -80,7 +81,7 @@ def load_pretrained_model(
 
     from ..utils import download_file
 
-    tokenizer = AutoTokenizer.from_pretrained(tokenizer_path, use_fast=False, **tokenizer_dict)
+    tokenizer = AutoTokenizer.from_pretrained(tokenizer_path, **tokenizer_dict)
     # tokenizer.pad_token = tokenizer.unk_token # could be redundant
 
     model_path = download_file(model_path, cache_dir=cache_dir)
@@ -106,7 +107,8 @@ def load_pretrained_model(
         model_args.pretrain_mm_mlp_adapter = pretrain_mm_mlp_adapter  # important to set to correct path
 
         model.get_model().initialize_vision_modules(
-            model_args)  # This will load the CLIP vision encoder and MLP projector
+            model_args
+        )  # This will load the CLIP vision encoder and MLP projector
     else:
         model.resize_token_embeddings(len(tokenizer))  # perhaps not needed
 

evalscope/metrics/t2v_metrics/models/vqascore_models/vqa_model.py

@@ -8,8 +8,9 @@ from ..model import ScoreModel
 class VQAScoreModel(ScoreModel):
 
     @abstractmethod
-    def forward(self, images: List[str], texts: List[str], question_template: str,
-                answer_template: str) -> torch.Tensor:
+    def forward(
+        self, images: List[str], texts: List[str], question_template: str, answer_template: str
+    ) -> torch.Tensor:
         """Forward pass of the model to return n scores for n (image, text) pairs (in PyTorch Tensor)
         question_template: a string with optional {} to be replaced with the 'text'
         answer_template: a string with optional {} to be replaced with the 'text'

evalscope/models/__init__.py

@@ -4,38 +4,15 @@ from typing import TYPE_CHECKING
 from evalscope.utils.import_utils import _LazyModule
 
 if TYPE_CHECKING:
-    from .adapters import (BaseModelAdapter, BFCLAdapter, ChatGenerationModelAdapter, ContinuationLogitsModelAdapter,
-                           CustomModelAdapter, MultiChoiceModelAdapter, ServerModelAdapter, T2IModelAdapter,
-                           TauBenchAdapter, initialize_model_adapter)
-    from .custom import CustomModel, DummyCustomModel
-    from .local_model import LocalModel, get_local_model
-    from .register import get_model_adapter
+    from .model_apis import llm_ckpt, mockllm, openai_api
 
 else:
     _import_structure = {
-        'adapters': [
-            'BaseModelAdapter',
-            'initialize_model_adapter',
-            'ChatGenerationModelAdapter',
-            'ContinuationLogitsModelAdapter',
-            'MultiChoiceModelAdapter',
-            'CustomModelAdapter',
-            'ServerModelAdapter',
-            'T2IModelAdapter',
-            'TauBenchAdapter',
-            'BFCLAdapter',
-        ],
-        'custom': [
-            'CustomModel',
-            'DummyCustomModel',
-        ],
-        'local_model': [
-            'LocalModel',
-            'get_local_model',
-        ],
-        'register': [
-            'get_model_adapter',
-        ],
+        'model_apis': [
+            'openai_api',
+            'mockllm',
+            'llm_ckpt',
+        ]
     }
 
     import sys

evalscope/models/mockllm.py

@@ -0,0 +1,65 @@
+from typing import Any, Dict, Generator, Iterable, Iterator, List, Optional, Union
+
+from evalscope.api.dataset import Dataset
+from evalscope.api.messages import ChatMessage
+from evalscope.api.model import GenerateConfig, ModelAPI, ModelOutput
+from evalscope.api.tool import ToolChoice, ToolInfo
+from evalscope.utils.function_utils import thread_safe
+
+
+class MockLLM(ModelAPI):
+    """A mock implementation of the ModelAPI class for testing purposes.
+
+    Always returns default_output, unless you pass in a model_args
+    key "custom_outputs" with a value of an Iterable[ModelOutput]
+    """
+
+    default_output = 'Default output from mockllm/model'
+
+    outputs: Iterator[ModelOutput]
+
+    def __init__(
+        self,
+        model_name: str,
+        base_url: Optional[str] = None,
+        api_key: Optional[str] = None,
+        config: GenerateConfig = GenerateConfig(),
+        custom_outputs: Iterable[ModelOutput] = None,
+        **model_args: Dict[str, Any],
+    ) -> None:
+        super().__init__(model_name, base_url, api_key, config)
+        self.model_args = model_args
+        if custom_outputs is not None:
+            # We cannot rely on the user of this model giving custom_outputs
+            # the correct type since they do not call this constructor
+            # Hence this type check and the one in generate.
+            if not isinstance(custom_outputs, (Iterable, Generator)):
+                raise ValueError(
+                    f"model_args['custom_outputs'] must be an Iterable or a Generator, got {custom_outputs}"
+                )
+            self.outputs = iter(custom_outputs)
+        else:
+            self.outputs = iter((
+                ModelOutput.from_content(model='mockllm', content=self.default_output)
+                for _ in iter(int, 1)  # produce an infinite iterator
+            ))
+
+    @thread_safe
+    def generate(
+        self,
+        input: List[ChatMessage],
+        tools: List[ToolInfo],
+        tool_choice: ToolChoice,
+        config: GenerateConfig,
+    ) -> ModelOutput:
+        try:
+            output = next(self.outputs)
+        except StopIteration:
+            raise ValueError('custom_outputs ran out of values')
+
+        if not isinstance(output, ModelOutput):
+            raise ValueError(f'output must be an instance of ModelOutput; got {type(output)}; content: {repr(output)}')
+        return output
+
+    def batch_generate(inputs: Dataset, config: GenerateConfig) -> List[ModelOutput]:
+        return super().batch_generate(inputs, config)

evalscope/models/model_apis.py

@@ -0,0 +1,47 @@
+from evalscope.api.model import ModelAPI
+from evalscope.api.registry import register_model_api
+from evalscope.utils.deprecation_utils import deprecated
+
+
+@register_model_api(name='mock_llm')
+def mockllm() -> type[ModelAPI]:
+    from .mockllm import MockLLM
+
+    return MockLLM
+
+
+@register_model_api(name='openai_api')
+def openai_api() -> type[ModelAPI]:
+    from .openai_compatible import OpenAICompatibleAPI
+
+    return OpenAICompatibleAPI
+
+
+@register_model_api(name='server')
+@deprecated(since='1.0.0', remove_in='1.1.0', alternative='openai_api')
+def server() -> type[ModelAPI]:
+    from .openai_compatible import OpenAICompatibleAPI
+
+    return OpenAICompatibleAPI
+
+
+@register_model_api(name='llm_ckpt')
+def llm_ckpt() -> type[ModelAPI]:
+    from .modelscope import ModelScopeAPI
+
+    return ModelScopeAPI
+
+
+@register_model_api(name='checkpoint')
+@deprecated(since='1.0.0', remove_in='1.1.0', alternative='llm_ckpt')
+def checkpoint() -> type[ModelAPI]:
+    from .modelscope import ModelScopeAPI
+
+    return ModelScopeAPI
+
+
+@register_model_api(name='text2image')
+def text2image() -> type[ModelAPI]:
+    from .text2image_model import Text2ImageAPI
+
+    return Text2ImageAPI

evalscope/models/modelscope.py

@@ -0,0 +1,455 @@
+from __future__ import annotations
+
+import copy
+import functools
+import json
+import time
+import torch  # type: ignore
+from concurrent.futures import Future
+from dataclasses import dataclass
+from logging import getLogger
+from modelscope import AutoModelForCausalLM, AutoTokenizer
+from queue import Empty, Queue
+from threading import Thread
+from torch import Tensor  # type: ignore
+from typing import Any, Dict, List, Literal, Optional, Protocol, Tuple, Union, cast
+from typing_extensions import override
+
+from evalscope.api.messages import (
+    ChatMessage,
+    ChatMessageAssistant,
+    ContentAudio,
+    ContentImage,
+    ContentText,
+    ContentVideo,
+)
+from evalscope.api.model import (
+    ChatCompletionChoice,
+    GenerateConfig,
+    Logprob,
+    Logprobs,
+    ModelAPI,
+    ModelOutput,
+    ModelUsage,
+    TopLogprob,
+)
+from evalscope.api.tool import ToolChoice, ToolInfo
+from evalscope.utils.model_utils import get_device
+
+logger = getLogger()
+
+
+class ModelScopeAPI(ModelAPI):
+
+    def __init__(
+        self,
+        model_name: str,
+        base_url: Optional[str] = None,
+        api_key: Optional[str] = None,
+        config: GenerateConfig = GenerateConfig(),
+        **model_args: Any,
+    ):
+        super().__init__(
+            model_name=model_name,
+            base_url=base_url,
+            api_key=api_key,
+            config=config,
+        )
+
+        # collect known model_args (then delete them so we can pass the rest on)
+        def collect_model_arg(name: str) -> Optional[Any]:
+            nonlocal model_args
+            value = model_args.get(name, None)
+            if value is not None:
+                model_args.pop(name)
+            return value
+
+        model_path = collect_model_arg('model_path')
+        device_map = collect_model_arg('device_map')
+        torch_dtype = collect_model_arg('precision')
+        tokenizer_path = collect_model_arg('tokenizer_path')
+        self.chat_template = collect_model_arg('chat_template')
+        self.tokenizer_call_args = collect_model_arg('tokenizer_call_args')
+        self.enable_thinking = collect_model_arg('enable_thinking')
+        if self.tokenizer_call_args is None:
+            self.tokenizer_call_args = {}
+
+        # device
+        self.device = device_map or get_device()
+
+        # torch dtype
+        DTYPE_MAP = {'float16': torch.float16, 'float32': torch.float32, 'bfloat16': torch.bfloat16, 'auto': 'auto'}
+
+        if isinstance(torch_dtype, str) and torch_dtype != 'auto':
+            torch_dtype = DTYPE_MAP.get(torch_dtype, torch.float32)
+        self.torch_dtype = torch_dtype
+
+        # model
+        model_name_or_path = model_path or model_name
+        self.model = AutoModelForCausalLM.from_pretrained(
+            model_name_or_path,
+            device_map=self.device,
+            token=self.api_key,
+            torch_dtype=self.torch_dtype,
+            trust_remote_code=True,
+            **model_args
+        )
+
+        # tokenizer
+        tokenizer_name_or_path = tokenizer_path or model_name_or_path
+        self.tokenizer = AutoTokenizer.from_pretrained(tokenizer_name_or_path, trust_remote_code=True)
+        # LLMs generally don't have a pad token and we need one for batching
+        if self.tokenizer.pad_token is None:
+            if self.tokenizer.eos_token is not None:
+                self.tokenizer.pad_token = self.tokenizer.eos_token
+            else:
+                # add a pad token
+                self.tokenizer.add_special_tokens({'pad_token': '[PAD]'})
+        # set padding side to left for LLMs
+        self.tokenizer.padding_side = 'left'
+        # set chat template if provided
+        if self.chat_template:
+            self.tokenizer.chat_template = self.chat_template
+            logger.info(f'Using custom chat template: {self.chat_template}')
+
+    def generate(
+        self,
+        input: List[ChatMessage],
+        tools: List[ToolInfo],
+        tool_choice: ToolChoice,
+        config: GenerateConfig,
+    ) -> ModelOutput:
+
+        # create chat
+        chat = self.ms_chat(input, tools)
+
+        assert isinstance(self.tokenizer_call_args, dict)
+        # prepare tokenizer
+        tokenizer = functools.partial(
+            self.tokenizer,
+            return_tensors='pt',
+            padding=True,
+            **self.tokenizer_call_args,
+        )
+
+        # prepare generator
+        kwargs: Dict[str, Any] = {}
+        if config.do_sample is not None:
+            kwargs['do_sample'] = config.do_sample
+        if config.n is not None:
+            if config.n > 1:
+                assert config.do_sample, 'n > 1 requires do_sample=True in GenerateConfig'
+            kwargs['num_return_sequences'] = config.n
+        if config.max_tokens is not None:
+            kwargs['max_new_tokens'] = config.max_tokens
+        if config.temperature is not None:
+            kwargs['temperature'] = config.temperature
+        if config.top_p is not None:
+            kwargs['top_p'] = config.top_p
+        if config.top_k is not None:
+            kwargs['top_k'] = config.top_k
+        if config.logprobs is not None:
+            kwargs['output_logits'] = config.logprobs
+        if 'return_dict_in_generate' in kwargs:
+            assert kwargs['return_dict_in_generate']
+        if config.stop_seqs is not None:
+            from transformers.generation import StopStringCriteria  # type: ignore
+
+            stopping_criteria = [StopStringCriteria(self.tokenizer, config.stop_seqs)]
+            kwargs['stopping_criteria'] = stopping_criteria
+
+        kwargs['return_dict_in_generate'] = True
+        generator = functools.partial(self.model.generate, **kwargs)
+
+        # prepare decoder
+        decoder = functools.partial(
+            self.tokenizer.batch_decode,
+            skip_special_tokens=True,
+            clean_up_tokenization_spaces=False,
+        )
+
+        # generate
+        responses = batched_generate(
+            GenerateInput(
+                input=chat,
+                device=self.model.device,
+                tokenizer=tokenizer,
+                generator=generator,
+                decoder=decoder,
+                batch_size=config.batch_size or self.max_connections(),
+            )
+        )
+
+        choices: List[ChatCompletionChoice] = []
+        for response in responses:
+            # gather logprobs
+            final_logprobs = None
+            if config.logprobs is not None:
+                final_logprobs = extract_logprobs(
+                    response=response,
+                    top=config.top_logprobs,
+                    tokenizer=self.tokenizer,
+                )
+
+            # construct choice
+            # TODO: Handle tool calls
+            choice = ChatCompletionChoice(
+                message=ChatMessageAssistant(content=response.output, model=self.model_name, source='generate'),
+                logprobs=(Logprobs(content=final_logprobs) if final_logprobs is not None else None),
+            )
+            choices.append(choice)
+
+        # return output
+        return ModelOutput(
+            model=self.model_name,
+            choices=choices,
+            usage=ModelUsage(
+                input_tokens=response.input_tokens,
+                output_tokens=response.output_tokens,
+                total_tokens=response.total_tokens,
+            ),
+            time=response.time,
+        )
+
+    @override
+    def max_tokens(self) -> Optional[int]:
+        """Default is 2048, bump it up to a value suitable for evals."""
+        return 2048
+
+    @override
+    def max_connections(self) -> int:
+        """Effectively the batch size."""
+        return 8
+
+    def ms_chat(self, messages: List[ChatMessage], tools: List[ToolInfo]) -> str:
+        # convert to ms format
+        tools_list = []
+        ms_messages = copy.deepcopy(messages)
+        if len(tools) > 0:
+            tools_list = [json.loads(tool.model_dump_json(exclude_none=True, indent=2)) for tool in tools]
+
+        ms_messages = message_content_to_string(ms_messages)
+        # apply chat template
+        if self.tokenizer.chat_template is not None:
+            chat = self.tokenizer.apply_chat_template(
+                ms_messages,
+                add_generation_prompt=True,
+                tokenize=False,
+                tools=tools_list if len(tools_list) > 0 else None,
+                enable_thinking=self.enable_thinking,  # not all models use this, check if it is supported
+            )
+        else:
+            chat = ''
+            for message in ms_messages:
+                chat += f'{message.role}: {message.content}\n'
+        # return
+        return cast(str, chat)
+
+
+def message_content_to_string(messages: List[ChatMessage]) -> List[ChatMessage]:
+    """Convert list of content in `ChatMessageAssistant`, `ChatMessageUser` or `ChatMessageSystem` to a string."""
+    for message in messages:
+        if isinstance(message.content, list):
+            is_multimodal = any(
+                isinstance(item, (ContentAudio, ContentImage, ContentVideo)) for item in message.content
+            )
+            if is_multimodal:
+                raise NotImplementedError(
+                    'Transformer model does not support multimodal content, please provide text inputs only.'
+                )
+            message.content = message.text
+    return messages
+
+
+# return value from generate as a result of specifying return_dict_in_generate
+class ModelGenerateOutput:
+    sequences: Tensor
+    logits: tuple[Tensor]
+
+
+class Tokenizer(Protocol):
+
+    def __call__(self, input: List[str]) -> Dict[Literal['input_ids', 'attention_mask'], Tensor]:
+        ...
+
+
+class Generator(Protocol):
+
+    def __call__(self, input_ids: Tensor, attention_mask: Tensor) -> Tensor:
+        ...
+
+
+class Decoder(Protocol):
+
+    def __call__(self, sequences: Tensor) -> list[str]:
+        ...
+
+
+@dataclass
+class GenerateInput:
+    input: str
+    device: str
+    tokenizer: Tokenizer
+    generator: Generator
+    decoder: Decoder
+    batch_size: int
+
+
+@dataclass
+class GenerateOutput:
+    output: str
+    input_tokens: int
+    output_tokens: int
+    total_tokens: int
+    logprobs: Optional[torch.Tensor]
+    time: float
+
+
+@dataclass
+class _QueueItem:
+    input: GenerateInput
+    future: Future[GenerateOutput]
+
+
+batch_thread: Optional[Thread] = None
+
+batch_queue: 'Queue[_QueueItem]' = Queue()
+
+
+def batched_generate(input: GenerateInput) -> List[GenerateOutput]:
+    # start the background thread if necessary
+    global batch_thread
+    if batch_thread is None:
+        batch_thread = Thread(target=process_batches, daemon=True)
+        batch_thread.start()
+
+    # enqueue the job
+    future = Future[GenerateOutput]()
+    batch_queue.put(_QueueItem(input=input, future=future))
+
+    return future.result()
+
+
+def process_batches() -> None:
+    while True:
+        # drain the queue (wait until no new messages have shown up for 2 seconds)
+        inputs: List[Tuple[GenerateInput, Future[GenerateOutput]]] = []
+        while True:
+            try:
+                input = batch_queue.get(timeout=2)
+                inputs.append((input.input, input.future))
+                if len(inputs) == input.input.batch_size:
+                    # max batch size reached
+                    break
+            except Empty:
+                # we have exhausted the queue
+                break
+
+        # see if we have any work to do
+        if len(inputs) == 0:
+            continue
+
+        try:
+            # capture the generator and decoder functions
+            start_time = time.monotonic()
+            first_input = inputs[0][0]
+            device = first_input.device
+            tokenizer = first_input.tokenizer
+            generator = first_input.generator
+            decoder = first_input.decoder
+            num_return_sequences = generator.keywords.get('num_return_sequences', 1)
+
+            # tokenize and move to device
+            tokenized_inputs = tokenizer([item[0].input for item in inputs])
+            input_ids = tokenized_inputs['input_ids']
+            attention_mask = tokenized_inputs['attention_mask']
+            input_ids = input_ids.to(device)
+            attention_mask = attention_mask.to(device)
+
+            # generate
+            with torch.inference_mode():
+                generation_outputs = cast(
+                    ModelGenerateOutput,
+                    generator(input_ids=input_ids, attention_mask=attention_mask),
+                )
+                generate_ids = generation_outputs.sequences
+                logits = generation_outputs.logits
+
+            # get logprobs from logits
+            logprobs = None
+            if logits is not None:
+                stacked_logits = torch.stack(logits).transpose(0, 1)
+                logprobs = torch.nn.functional.log_softmax(stacked_logits, dim=-1)
+
+            # decode
+            generated_tokens = generate_ids[:, input_ids.size(dim=1):]
+            if logprobs is not None:
+                assert logprobs.shape[1] == generated_tokens.shape[1]
+            outputs = decoder(sequences=generated_tokens)
+
+            # call back futures
+            total_time = time.monotonic() - start_time
+            for input_index in range(len(inputs)):
+                choices: List[GenerateOutput] = []
+                # handle input
+                future = inputs[input_index][1]
+                input_tokens = input_ids[input_index].shape[-1]
+                # handle choices
+                for choice_index in range(num_return_sequences):
+                    output_index = input_index * num_return_sequences + choice_index
+                    # handle out of
+                    output = outputs[output_index]
+                    output_tokens = generate_ids[output_index].shape[-1] - input_tokens
+                    logprobs_tensor = logprobs[output_index] if logprobs is not None else None
+                    # create the output
+                    choices.append(
+                        GenerateOutput(
+                            output=output,
+                            input_tokens=input_tokens,
+                            output_tokens=output_tokens,
+                            total_tokens=input_tokens + output_tokens,
+                            logprobs=logprobs_tensor,
+                            time=total_time,
+                        )
+                    )
+
+                # asyncio futures are not thread safe, so we need to pass the event loop
+                # down to this point, so we can mark the future as done in a thread safe manner.
+                # see: https://docs.python.org/3/library/asyncio-dev.html#concurrency-and-multithreading
+                future.set_result(choices)
+
+        except Exception as ex:
+            for inp in inputs:
+                future = inp[1]
+                future.set_exception(ex)
+
+
+def extract_logprobs(
+    response: GenerateOutput,
+    top: Optional[int],
+    tokenizer,
+) -> List[Logprob]:
+    assert response.logprobs is not None
+    k = top or 1
+    topk_values, topk_inds = response.logprobs.topk(k=k, dim=-1)
+    final_logprobs = []
+    for toks, vals in zip(topk_inds, topk_values):
+        top_logprobs: List[TopLogprob] = []
+        for tok, val in zip(toks, vals):
+            # TODO: you get byte artifacts converting single ids to tokens like this...
+            # but `tokenizer.decode` strips spaces. There must be a better way to do this.
+            token_str = tokenizer.convert_ids_to_tokens(tok.item())
+            top_logprobs.append(TopLogprob(
+                token=token_str,
+                logprob=val,
+                bytes=list(map(ord, token_str)),
+            ))
+        final_logprobs.append(
+            Logprob(
+                token=top_logprobs[0].token,
+                logprob=top_logprobs[0].logprob,
+                bytes=top_logprobs[0].bytes,
+                top_logprobs=top_logprobs,
+            )
+        )
+    return final_logprobs