evalscope 0.10.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

evalscope/metrics/t2v_metrics/models/vqascore_models/clip_t5/model/language_model/clip_t5.py

@@ -0,0 +1,306 @@
+#    Copyright 2023 Zhiqiu Lin
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+
+import torch
+from dataclasses import dataclass, field
+from transformers import AutoConfig, AutoModelForSeq2SeqLM, T5Config, T5ForConditionalGeneration
+from transformers.modeling_outputs import Seq2SeqLMOutput
+from typing import List, Optional, Tuple, Union
+
+from ..multimodal_encoder.builder import build_vision_tower
+from ..multimodal_projector.builder import build_vision_projector
+
+IMAGE_TOKEN_INDEX = -200
+
+
+@dataclass
+class ModelArguments:
+    tune_mm_mlp_adapter: bool = field(default=False)
+    vision_tower: Optional[str] = field(default='openai/clip-vit-large-patch14-336')
+    mm_vision_select_layer: Optional[int] = field(default=-2)  # default to the second last layer in llava1.5
+    pretrain_mm_mlp_adapter: Optional[str] = field(default=None)
+    mm_projector_type: Optional[str] = field(default='mlp2x_gelu')
+    mm_vision_select_feature: Optional[str] = field(default='patch')
+
+
+class CLIPT5Config(T5Config):
+    model_type = 'clip_t5'
+
+
+class CLIPT5ForConditionalGeneration(T5ForConditionalGeneration):
+    # This class supports both T5 and FlanT5
+    config_class = CLIPT5Config
+
+    def __init__(self, config):
+        super(CLIPT5ForConditionalGeneration, self).__init__(config)
+        self.embed_tokens = self.encoder.embed_tokens
+        if hasattr(config, 'mm_vision_tower'):
+            self.vision_tower = build_vision_tower(config, delay_load=False)
+            self.mm_projector = build_vision_projector(config)
+
+    def get_vision_tower(self):
+        vision_tower = getattr(self, 'vision_tower', None)
+        if type(vision_tower) is list:
+            vision_tower = vision_tower[0]
+        return vision_tower
+
+    def get_model(self):
+        return self  # for compatibility with LlavaMetaForCausalLM
+
+    def prepare_inputs_labels_for_multimodal(
+        self, input_ids, attention_mask, decoder_attention_mask, past_key_values, labels, images
+    ):
+        # The labels are now separated from the input_ids.
+        vision_tower = self.get_vision_tower()
+        if vision_tower is None or images is None or input_ids.shape[1] == 1:
+            raise NotImplementedError()
+
+        if type(images) is list or images.ndim == 5:
+            concat_images = torch.cat([image for image in images], dim=0)
+            image_features = self.encode_images(concat_images)
+            split_sizes = [image.shape[0] for image in images]
+            image_features = torch.split(image_features, split_sizes, dim=0)
+            image_features = [x.flatten(0, 1) for x in image_features]
+        else:
+            image_features = self.encode_images(images)
+
+        new_input_embeds = []
+        cur_image_idx = 0
+        for _, cur_input_ids in enumerate(input_ids):
+            if (cur_input_ids == IMAGE_TOKEN_INDEX).sum() == 0:
+                # multimodal LLM, but the current sample is not multimodal
+                raise NotImplementedError()
+            image_token_indices = torch.where(cur_input_ids == IMAGE_TOKEN_INDEX)[0]
+            cur_new_input_embeds = []
+            while image_token_indices.numel() > 0:
+                cur_image_features = image_features[cur_image_idx]
+                image_token_start = image_token_indices[0]
+                cur_new_input_embeds.append(self.embed_tokens(cur_input_ids[:image_token_start]))
+                cur_new_input_embeds.append(cur_image_features)
+                cur_image_idx += 1
+                cur_input_ids = cur_input_ids[image_token_start + 1:]
+                image_token_indices = torch.where(cur_input_ids == IMAGE_TOKEN_INDEX)[0]
+            if cur_input_ids.numel() > 0:
+                cur_new_input_embeds.append(self.embed_tokens(cur_input_ids))
+            cur_new_input_embeds = [x.to(device=self.device) for x in cur_new_input_embeds]
+            cur_new_input_embeds = torch.cat(cur_new_input_embeds, dim=0)
+            new_input_embeds.append(cur_new_input_embeds)
+
+        if any(x.shape != new_input_embeds[0].shape for x in new_input_embeds):
+            max_len = max(x.shape[0] for x in new_input_embeds)
+
+            new_input_embeds_align = []
+            _input_embeds_lengths = []
+            for cur_new_embed in new_input_embeds:
+                _input_embeds_lengths.append(cur_new_embed.shape[0])
+                cur_new_embed = torch.cat((
+                    cur_new_embed,
+                    torch.zeros((max_len - cur_new_embed.shape[0], cur_new_embed.shape[1]),
+                                dtype=cur_new_embed.dtype,
+                                device=cur_new_embed.device)
+                ),
+                                          dim=0)
+                new_input_embeds_align.append(cur_new_embed)
+            new_input_embeds = torch.stack(new_input_embeds_align, dim=0)
+
+            if attention_mask is not None:
+                new_attention_mask = []
+                for cur_attention_mask, _input_embeds_length in zip(attention_mask, _input_embeds_lengths):
+                    new_attn_mask_pad_left = torch.full((_input_embeds_length - input_ids.shape[1], ),
+                                                        True,
+                                                        dtype=attention_mask.dtype,
+                                                        device=attention_mask.device)
+                    new_attn_mask_pad_right = torch.full((new_input_embeds.shape[1] - _input_embeds_length, ),
+                                                         False,
+                                                         dtype=attention_mask.dtype,
+                                                         device=attention_mask.device)
+                    cur_new_attention_mask = torch.cat(
+                        (new_attn_mask_pad_left, cur_attention_mask, new_attn_mask_pad_right), dim=0
+                    )
+                    new_attention_mask.append(cur_new_attention_mask)
+                attention_mask = torch.stack(new_attention_mask, dim=0)
+                assert attention_mask.shape == new_input_embeds.shape[:2]
+        else:
+            new_input_embeds = torch.stack(new_input_embeds, dim=0)
+
+            if attention_mask is not None:
+                new_attn_mask_pad_left = torch.full(
+                    (attention_mask.shape[0], new_input_embeds.shape[1] - input_ids.shape[1]),
+                    True,
+                    dtype=attention_mask.dtype,
+                    device=attention_mask.device
+                )
+                attention_mask = torch.cat((new_attn_mask_pad_left, attention_mask), dim=1)
+                assert attention_mask.shape == new_input_embeds.shape[:2]
+
+        return None, attention_mask, decoder_attention_mask, past_key_values, new_input_embeds, labels
+
+    def encode_images(self, images):
+        image_features = self.get_vision_tower()(images)
+        image_features = self.mm_projector(image_features)
+        return image_features
+
+    def initialize_vision_modules(self, model_args, fsdp=None):
+        vision_tower = model_args.vision_tower
+        mm_vision_select_layer = model_args.mm_vision_select_layer
+        mm_vision_select_feature = model_args.mm_vision_select_feature
+        pretrain_mm_mlp_adapter = model_args.pretrain_mm_mlp_adapter
+
+        self.config.mm_vision_tower = vision_tower
+        self.config.pretrain_mm_mlp_adapter = pretrain_mm_mlp_adapter
+
+        if self.get_vision_tower() is None:
+            vision_tower = build_vision_tower(model_args)
+
+            if fsdp is not None and len(fsdp) > 0:
+                self.vision_tower = [vision_tower]
+            else:
+                self.vision_tower = vision_tower
+        else:
+            if fsdp is not None and len(fsdp) > 0:
+                vision_tower = self.vision_tower[0]
+            else:
+                vision_tower = self.vision_tower
+            if not vision_tower.is_loaded:
+                vision_tower.load_model()
+
+        self.config.use_mm_proj = True
+        self.config.mm_projector_type = getattr(model_args, 'mm_projector_type', 'mlp2x_gelu')
+        self.config.mm_hidden_size = vision_tower.hidden_size
+        self.config.mm_vision_select_layer = mm_vision_select_layer
+        self.config.mm_vision_select_feature = mm_vision_select_feature
+
+        if getattr(self, 'mm_projector', None) is None:
+            self.mm_projector = build_vision_projector(self.config)
+
+        if pretrain_mm_mlp_adapter is not None:
+            mm_projector_weights = torch.load(pretrain_mm_mlp_adapter, map_location='cpu')
+
+            def get_w(weights, keyword):
+                return {k.split(keyword + '.')[1]: v for k, v in weights.items() if keyword in k}
+
+            self.mm_projector.load_state_dict(get_w(mm_projector_weights, 'mm_projector'))
+
+    def forward(
+        self,
+        input_ids: torch.LongTensor = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        decoder_attention_mask: Optional[torch.Tensor] = None,
+        past_key_values: Optional[List[torch.FloatTensor]] = None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        labels: Optional[torch.LongTensor] = None,
+        use_cache: Optional[bool] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        images: Optional[torch.FloatTensor] = None,
+        return_dict: Optional[bool] = None,
+        **kwargs,
+    ) -> Union[Tuple[torch.FloatTensor], Seq2SeqLMOutput]:
+        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+        output_hidden_states = (
+            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+        )
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+        if inputs_embeds is None:
+            _, attention_mask, decoder_attention_mask, past_key_values, inputs_embeds, labels = \
+                self.prepare_inputs_labels_for_multimodal(input_ids, attention_mask, decoder_attention_mask, past_key_values, labels, images)
+
+        # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn)
+        outputs = super(CLIPT5ForConditionalGeneration, self).forward(
+            input_ids=None,  # will be None if inputs_embeds is not None
+            attention_mask=attention_mask,
+            decoder_attention_mask=decoder_attention_mask,
+            labels=labels,
+            past_key_values=past_key_values,
+            inputs_embeds=inputs_embeds,
+            use_cache=use_cache,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+            **kwargs,
+        )
+
+        return outputs
+
+    @torch.no_grad()
+    def generate(
+        self,
+        inputs: Optional[torch.Tensor] = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        images: Optional[torch.Tensor] = None,
+        **kwargs,
+    ):
+        assert images is not None, 'images must be provided'
+        assert inputs is not None, 'inputs must be provided'
+        assert attention_mask is not None, 'attention_mask must be provided'
+        _, attention_mask, _, _, inputs_embeds, _ = \
+            self.prepare_inputs_labels_for_multimodal(inputs, attention_mask, None, None, None, images)
+        # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn)
+        outputs = super(CLIPT5ForConditionalGeneration, self).generate(
+            input_ids=None,  # will be None if inputs_embeds is not None
+            attention_mask=attention_mask,
+            inputs_embeds=inputs_embeds,
+        )
+        return outputs
+
+    def prepare_inputs_for_generation(
+        self,
+        input_ids,
+        past_key_values=None,
+        attention_mask=None,
+        head_mask=None,
+        decoder_head_mask=None,
+        decoder_attention_mask=None,
+        cross_attn_head_mask=None,
+        use_cache=None,
+        encoder_outputs=None,
+        inputs_embeds=None,
+        **kwargs,
+    ):
+        # cut decoder_input_ids if past_key_values is used
+        if past_key_values is not None:
+            past_length = past_key_values[0][0].shape[2]
+
+            # Some generation methods already pass only the last input ID
+            if input_ids.shape[1] > past_length:
+                remove_prefix_length = past_length
+            else:
+                # Default to old behavior: keep only final ID
+                remove_prefix_length = input_ids.shape[1] - 1
+
+            input_ids = input_ids[:, remove_prefix_length:]
+
+        # if `inputs_embeds` are passed, we only want to use them in the 1st generation step
+        if inputs_embeds is not None and past_key_values is None:
+            model_inputs = {'inputs_embeds': inputs_embeds}
+        else:
+            model_inputs = {'input_ids': input_ids}
+
+        model_inputs.update({
+            'decoder_input_ids': input_ids,
+            'past_key_values': past_key_values,
+            'encoder_outputs': encoder_outputs,
+            'attention_mask': attention_mask,
+            'head_mask': head_mask,
+            'decoder_head_mask': decoder_head_mask,
+            'decoder_attention_mask': decoder_attention_mask,
+            'cross_attn_head_mask': cross_attn_head_mask,
+            'use_cache': use_cache,
+        })
+        return model_inputs
+
+
+AutoConfig.register('clip_t5', CLIPT5Config)
+AutoModelForSeq2SeqLM.register(CLIPT5Config, CLIPT5ForConditionalGeneration)

evalscope/metrics/t2v_metrics/models/vqascore_models/clip_t5/model/multimodal_encoder/builder.py

@@ -0,0 +1,12 @@
+import os
+
+from .clip_encoder import CLIPVisionTower
+
+
+def build_vision_tower(vision_tower_cfg, **kwargs):
+    vision_tower = getattr(vision_tower_cfg, 'mm_vision_tower', getattr(vision_tower_cfg, 'vision_tower', None))
+    is_absolute_path_exists = os.path.exists(vision_tower)
+    if is_absolute_path_exists or vision_tower.startswith('openai') or vision_tower.startswith('laion'):
+        return CLIPVisionTower(vision_tower, args=vision_tower_cfg, **kwargs)
+
+    raise ValueError(f'Unknown vision tower: {vision_tower}')

evalscope/metrics/t2v_metrics/models/vqascore_models/clip_t5/model/multimodal_encoder/clip_encoder.py

@@ -0,0 +1,84 @@
+import torch
+import torch.nn as nn
+from transformers import CLIPImageProcessor, CLIPVisionConfig, CLIPVisionModel
+
+
+class CLIPVisionTower(nn.Module):
+
+    def __init__(self, vision_tower, args, delay_load=False):
+        super().__init__()
+
+        self.is_loaded = False
+
+        self.vision_tower_name = vision_tower
+        self.select_layer = args.mm_vision_select_layer
+        self.select_feature = getattr(args, 'mm_vision_select_feature', 'patch')
+
+        if not delay_load:
+            self.load_model()
+        else:
+            self.cfg_only = CLIPVisionConfig.from_pretrained(self.vision_tower_name)
+
+    def load_model(self):
+        from .....utils import download_file
+        model_path = download_file(self.vision_tower_name.replace('openai', 'openai-mirror'))
+        self.image_processor = CLIPImageProcessor.from_pretrained(model_path)
+        self.vision_tower = CLIPVisionModel.from_pretrained(model_path)
+        self.vision_tower.requires_grad_(False)
+
+        self.is_loaded = True
+
+    def feature_select(self, image_forward_outs):
+        image_features = image_forward_outs.hidden_states[self.select_layer]
+        if self.select_feature == 'patch':
+            image_features = image_features[:, 1:]
+        elif self.select_feature == 'cls_patch':
+            image_features = image_features
+        else:
+            raise ValueError(f'Unexpected select feature: {self.select_feature}')
+        return image_features
+
+    @torch.no_grad()
+    def forward(self, images):
+        if type(images) is list:
+            image_features = []
+            for image in images:
+                image_forward_out = self.vision_tower(
+                    image.to(device=self.device, dtype=self.dtype).unsqueeze(0), output_hidden_states=True
+                )
+                image_feature = self.feature_select(image_forward_out).to(image.dtype)
+                image_features.append(image_feature)
+        else:
+            image_forward_outs = self.vision_tower(
+                images.to(device=self.device, dtype=self.dtype), output_hidden_states=True
+            )
+            image_features = self.feature_select(image_forward_outs).to(images.dtype)
+
+        return image_features
+
+    @property
+    def dummy_feature(self):
+        return torch.zeros(1, self.hidden_size, device=self.device, dtype=self.dtype)
+
+    @property
+    def dtype(self):
+        return self.vision_tower.dtype
+
+    @property
+    def device(self):
+        return self.vision_tower.device
+
+    @property
+    def config(self):
+        if self.is_loaded:
+            return self.vision_tower.config
+        else:
+            return self.cfg_only
+
+    @property
+    def hidden_size(self):
+        return self.config.hidden_size
+
+    @property
+    def num_patches(self):
+        return (self.config.image_size // self.config.patch_size)**2

evalscope/metrics/t2v_metrics/models/vqascore_models/clip_t5/model/multimodal_projector/builder.py

@@ -0,0 +1,50 @@
+import re
+import torch
+import torch.nn as nn
+
+
+class IdentityMap(nn.Module):
+
+    def __init__(self):
+        super().__init__()
+
+    def forward(self, x, *args, **kwargs):
+        return x
+
+    @property
+    def config(self):
+        return {'mm_projector_type': 'identity'}
+
+
+class SimpleResBlock(nn.Module):
+
+    def __init__(self, channels):
+        super().__init__()
+        self.pre_norm = nn.LayerNorm(channels)
+
+        self.proj = nn.Sequential(nn.Linear(channels, channels), nn.GELU(), nn.Linear(channels, channels))
+
+    def forward(self, x):
+        x = self.pre_norm(x)
+        return x + self.proj(x)
+
+
+def build_vision_projector(config, delay_load=False, **kwargs):
+    projector_type = getattr(config, 'mm_projector_type', 'linear')
+
+    if projector_type == 'linear':
+        return nn.Linear(config.mm_hidden_size, config.hidden_size)
+
+    mlp_gelu_match = re.match(r'^mlp(\d+)x_gelu$', projector_type)
+    if mlp_gelu_match:
+        mlp_depth = int(mlp_gelu_match.group(1))
+        modules = [nn.Linear(config.mm_hidden_size, config.hidden_size)]
+        for _ in range(1, mlp_depth):
+            modules.append(nn.GELU())
+            modules.append(nn.Linear(config.hidden_size, config.hidden_size))
+        return nn.Sequential(*modules)
+
+    if projector_type == 'identity':
+        return IdentityMap()
+
+    raise ValueError(f'Unknown projector type: {projector_type}')

evalscope/metrics/t2v_metrics/models/vqascore_models/clip_t5_model.py

@@ -0,0 +1,223 @@
+import torch
+from typing import List
+
+from ...constants import CACHE_DIR, CONTEXT_LEN, DEFAULT_IMAGE_TOKEN, IGNORE_INDEX, SYSTEM_MSG
+from .clip_t5.model import CLIPT5ForConditionalGeneration, ModelArguments
+from .mm_utils import expand2square, load_pretrained_model, t5_tokenizer_image_token
+from .vqa_model import VQAScoreModel
+
+default_question_template = 'Does this figure show "{}"? Please answer yes or no.'
+default_answer_template = 'Yes'
+
+
+def format_question(question, conversation_style='plain'):
+    if conversation_style == 't5_plain':  # for 1st stage t5 model
+        question = DEFAULT_IMAGE_TOKEN + question
+    elif conversation_style == 't5_chat':  # for 2nd stage t5 model
+        question = SYSTEM_MSG + ' USER: ' + DEFAULT_IMAGE_TOKEN + '\n' + question + ' ASSISTANT: '
+    elif conversation_style == 't5_chat_no_system':  # for 2nd stage t5 model
+        question = 'USER: ' + DEFAULT_IMAGE_TOKEN + '\n' + question + ' ASSISTANT: '
+    elif conversation_style == 't5_chat_no_system_no_user':  # for 2nd stage t5 model
+        question = '' + DEFAULT_IMAGE_TOKEN + '\n' + question + ' : '
+    # elif conversation_style == 't5_chat_ood_system': # for 2nd stage t5 model
+    #     question = SYSTEM_MSG + " HUMAN: " + DEFAULT_IMAGE_TOKEN + "\n" + question + " GPT: "
+    else:
+        raise NotImplementedError()
+    return question
+
+
+def format_answer(answer, conversation_style='plain'):
+    return answer
+
+
+CLIP_T5_MODELS = {
+    # We recommend using 'clip-flant5-xxl' for maximal performance.
+    # If you want to use a smaller model, we recommend using 'clip-flant5-xl'.
+    'clip-flant5-xxl': {
+        'tokenizer': {
+            'path': 'AI-ModelScope/clip-flant5-xxl',  # zhiqiulin/clip-flant5-xxl
+            'model_max_length': CONTEXT_LEN,
+        },
+        'model': {
+            'path': 'AI-ModelScope/clip-flant5-xxl',  # zhiqiulin/clip-flant5-xxl
+            'conversation': 't5_chat',
+            'image_aspect_ratio': 'pad',
+        },
+    },
+    'clip-flant5-xl': {
+        'tokenizer': {
+            'path': 'zhiqiulin/clip-flant5-xl',
+            'model_max_length': CONTEXT_LEN,
+        },
+        'model': {
+            'path': 'zhiqiulin/clip-flant5-xl',
+            'conversation': 't5_chat',
+            'image_aspect_ratio': 'pad',
+        },
+    },
+}
+
+
+class CLIPT5Model(VQAScoreModel):
+    """A wrapper for the CLIP-FlanT5 or CLIP-T5 models"""
+
+    def __init__(self, model_name='clip-flant5-xxl', device='cuda', cache_dir=CACHE_DIR):
+        assert model_name in CLIP_T5_MODELS
+        super().__init__(model_name=model_name, device=device, cache_dir=cache_dir)
+
+    def load_model(self):
+        """Load the model, tokenizer, image transform
+        """
+        model_args = ModelArguments()
+        model_max_length = CLIP_T5_MODELS[self.model_name]['tokenizer']['model_max_length'] \
+            if 'model_max_length' in CLIP_T5_MODELS[self.model_name]['tokenizer'] else None
+        padding_side = CLIP_T5_MODELS[self.model_name]['tokenizer']['padding_side'] \
+            if 'padding_side' in CLIP_T5_MODELS[self.model_name]['tokenizer'] else None
+        mmprojector_repo = CLIP_T5_MODELS[self.model_name]['model']['mmprojector_repo'] \
+            if 'mmprojector_repo' in CLIP_T5_MODELS[self.model_name]['model'] else None
+        mmprojector_name = CLIP_T5_MODELS[self.model_name]['model']['mmprojector_name'] \
+            if 'mmprojector_name' in CLIP_T5_MODELS[self.model_name]['model'] else None
+
+        # default is 'pad'
+        # stage-1 models use 'square'
+        self.image_aspect_ratio = CLIP_T5_MODELS[self.model_name]['model']['image_aspect_ratio'] \
+            if 'image_aspect_ratio' in CLIP_T5_MODELS[self.model_name]['model'] else 'pad'
+
+        self.conversational_style = CLIP_T5_MODELS[self.model_name]['model']['conversation']
+
+        self.context_len = CONTEXT_LEN
+
+        self.tokenizer, self.model, self.image_processor = load_pretrained_model(
+            CLIPT5ForConditionalGeneration,
+            model_args,
+            model_path=CLIP_T5_MODELS[self.model_name]['model']['path'],
+            tokenizer_path=CLIP_T5_MODELS[self.model_name]['tokenizer']['path'],
+            model_max_length=model_max_length,
+            padding_side=padding_side,
+            image_aspect_ratio=self.image_aspect_ratio,
+            mmprojector_repo=mmprojector_repo,
+            mmprojector_name=mmprojector_name,
+            device=self.device,
+            cache_dir=self.cache_dir
+        )
+
+    def load_images(self, image: List[str]) -> torch.Tensor:
+        """Load the image(s), and return a tensor (after preprocessing) put on self.device
+        """
+        image = [self.image_loader(x) for x in image]
+        if self.image_aspect_ratio == 'pad':
+            image = [
+                expand2square(image, tuple(int(x * 255) for x in self.image_processor.image_mean)) for image in image
+            ]
+        image = [self.image_processor.preprocess(image, return_tensors='pt')['pixel_values'][0] for image in image]
+        assert all(x.shape == image[0].shape for x in image)
+        image = torch.stack(image, dim=0).to(self.device)
+        return image
+
+    @torch.no_grad()
+    @torch.autocast(device_type='cuda', dtype=torch.bfloat16)
+    def forward(
+        self,
+        images: List[str],
+        texts: List[str],
+        question_template: str = default_question_template,
+        answer_template: str = default_answer_template
+    ) -> torch.Tensor:
+        """Forward pass of the model to return n scores for n (image, text) pairs (in PyTorch Tensor)
+        """
+        assert len(images) == len(texts), 'Number of images and texts must match'
+        # Turn "a photo of a dog" into
+        # Q: "Does this figure show "a photo of a dog"? Please answer yes or no."
+        # A: "Yes"
+        questions = [question_template.format(text) for text in texts]
+        answers = [answer_template.format(text) for text in texts]
+
+        # Formatting for CLIP-FlanT5 desired input including system message and image tokens
+        questions = [format_question(question, conversation_style=self.conversational_style) for question in questions]
+        answers = [format_answer(answer, conversation_style=self.conversational_style) for answer in answers]
+
+        images = self.load_images(images)
+
+        input_ids = [t5_tokenizer_image_token(qs, self.tokenizer, return_tensors='pt') for qs in questions]
+        labels = [t5_tokenizer_image_token(ans, self.tokenizer, return_tensors='pt') for ans in answers]
+
+        input_ids = torch.nn.utils.rnn.pad_sequence(
+            input_ids, batch_first=True, padding_value=self.tokenizer.pad_token_id
+        )
+        labels = torch.nn.utils.rnn.pad_sequence(labels, batch_first=True, padding_value=IGNORE_INDEX)
+        input_ids = input_ids[:, :self.tokenizer.model_max_length]
+        labels = labels[:, :self.tokenizer.model_max_length]
+
+        attention_mask = input_ids.ne(self.tokenizer.pad_token_id)
+        decoder_attention_mask = labels.ne(IGNORE_INDEX)
+
+        input_ids, attention_mask, decoder_attention_mask, labels = input_ids.to(self.device), \
+            attention_mask.to(self.device), decoder_attention_mask.to(self.device), labels.to(self.device)
+        model_input_kwargs = {
+            'input_ids': input_ids,
+            'attention_mask': attention_mask,
+            'decoder_attention_mask': decoder_attention_mask,
+            'labels': labels,
+            'images': images,
+            'past_key_values': None,
+            'inputs_embeds': None,
+            'use_cache': None,
+            'output_attentions': None,
+            'output_hidden_states': None,
+            'return_dict': True,
+        }
+
+        outputs = self.model(**model_input_kwargs)
+
+        logits = outputs.logits
+        lm_prob = torch.zeros(logits.shape[0])
+        loss_fct = torch.nn.CrossEntropyLoss(reduction='mean')
+        for k in range(lm_prob.shape[0]):
+            lm_prob[k] = (-loss_fct(logits[k],
+                                    labels[k])).exp()  # exp to cancel the log and get raw prob between 0 and 1
+        return lm_prob
+
+    @torch.no_grad()
+    @torch.autocast(device_type='cuda', dtype=torch.bfloat16)
+    def generate(
+        self,
+        images: List[str],
+        prompts: List[str],
+        temperature: float = 0.2,
+    ):
+        """Forward pass of the model to return n strings for n (image, prompt) pairs
+        """
+        assert len(images) == len(prompts), 'Number of images and texts must match'
+
+        # Formatting for CLIP-FlanT5 desired input including system message and image tokens
+        questions = [format_question(prompt, conversation_style=self.conversational_style) for prompt in prompts]
+        images = self.load_images(images)
+
+        input_ids = [t5_tokenizer_image_token(qs, self.tokenizer, return_tensors='pt') for qs in questions]
+        input_ids = torch.nn.utils.rnn.pad_sequence(
+            input_ids, batch_first=True, padding_value=self.tokenizer.pad_token_id
+        )
+        input_ids = input_ids[:, :self.tokenizer.model_max_length]
+
+        attention_mask = input_ids.ne(self.tokenizer.pad_token_id)
+
+        input_ids, attention_mask = input_ids.to(self.device), attention_mask.to(self.device)
+        model_input_kwargs = {
+            'inputs': input_ids,
+            'images': images,
+            'attention_mask': attention_mask,
+            'do_sample': True if temperature > 0 else False,
+            'temperature': temperature,
+            'top_p': None,
+            'num_beams': 1,
+            'max_new_token': 1024,
+            'use_cache': True,
+        }
+
+        outputs = self.model.generate(**model_input_kwargs)
+        outputs = self.tokenizer.batch_decode(outputs, skip_special_tokens=True)
+        for i in range(len(outputs)):
+            if outputs[i].endswith(' '):
+                outputs[i] = outputs[i][:-1]
+            outputs[i] = outputs[i].strip()
+        return outputs