evalscope 0.10.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

evalscope/metrics/t2v_metrics/models/vqascore_models/lavis/models/blip_models/__init__.py

@@ -0,0 +1,83 @@
+"""
+ Copyright (c) 2022, salesforce.com, inc.
+ All rights reserved.
+ SPDX-License-Identifier: BSD-3-Clause
+ For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
+"""
+
+import logging
+from torch import nn
+from typing import List
+
+
+def tie_encoder_decoder_weights(encoder: nn.Module, decoder: nn.Module, base_model_prefix: str, skip_key: str):
+    uninitialized_encoder_weights: List[str] = []
+    if decoder.__class__ != encoder.__class__:
+        logging.info(
+            f'{decoder.__class__} and {encoder.__class__} are not equal. In this case make sure that all encoder weights are correctly initialized.'
+        )
+
+    def tie_encoder_to_decoder_recursively(
+        decoder_pointer: nn.Module,
+        encoder_pointer: nn.Module,
+        module_name: str,
+        uninitialized_encoder_weights: List[str],
+        skip_key: str,
+        depth=0,
+    ):
+        assert isinstance(decoder_pointer, nn.Module) and isinstance(
+            encoder_pointer, nn.Module
+        ), f'{decoder_pointer} and {encoder_pointer} have to be of type torch.nn.Module'
+        if hasattr(decoder_pointer, 'weight') and skip_key not in module_name:
+            assert hasattr(encoder_pointer, 'weight')
+            encoder_pointer.weight = decoder_pointer.weight
+            if hasattr(decoder_pointer, 'bias'):
+                assert hasattr(encoder_pointer, 'bias')
+                encoder_pointer.bias = decoder_pointer.bias
+            # print(module_name + " is tied")
+            return
+
+        encoder_modules = encoder_pointer._modules
+        decoder_modules = decoder_pointer._modules
+        if len(decoder_modules) > 0:
+            assert (
+                len(encoder_modules) > 0
+            ), f'Encoder module {encoder_pointer} does not match decoder module {decoder_pointer}'
+
+            all_encoder_weights = set([module_name + '/' + sub_name for sub_name in encoder_modules.keys()])
+            encoder_layer_pos = 0
+            for name, module in decoder_modules.items():
+                if name.isdigit():
+                    encoder_name = str(int(name) + encoder_layer_pos)
+                    decoder_name = name
+                    if not isinstance(
+                        decoder_modules[decoder_name],
+                        type(encoder_modules[encoder_name]),
+                    ) and len(encoder_modules) != len(decoder_modules):
+                        # this can happen if the name corresponds to the position in a list module list of layers
+                        # in this case the decoder has added a cross-attention that the encoder does not have
+                        # thus skip this step and subtract one layer pos from encoder
+                        encoder_layer_pos -= 1
+                        continue
+                elif name not in encoder_modules:
+                    continue
+                elif depth > 500:
+                    raise ValueError(
+                        'Max depth of recursive function `tie_encoder_to_decoder` reached. It seems that there is a circular dependency between two or more `nn.Modules` of your model.'
+                    )
+                else:
+                    decoder_name = encoder_name = name
+                tie_encoder_to_decoder_recursively(
+                    decoder_modules[decoder_name],
+                    encoder_modules[encoder_name],
+                    module_name + '/' + name,
+                    uninitialized_encoder_weights,
+                    skip_key,
+                    depth=depth + 1,
+                )
+                all_encoder_weights.remove(module_name + '/' + encoder_name)
+
+            uninitialized_encoder_weights += list(all_encoder_weights)
+
+    # tie weights recursively
+    tie_encoder_to_decoder_recursively(decoder, encoder, base_model_prefix, uninitialized_encoder_weights, skip_key)

evalscope/metrics/t2v_metrics/models/vqascore_models/lavis/models/blip_models/blip.py

@@ -0,0 +1,58 @@
+"""
+ Copyright (c) 2022, salesforce.com, inc.
+ All rights reserved.
+ SPDX-License-Identifier: BSD-3-Clause
+ For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
+"""
+
+import logging
+import os
+import torch
+from modelscope import AutoTokenizer
+
+from ...common.dist_utils import download_cached_file
+from ...common.utils import is_url
+from ...models.base_model import BaseModel
+from ...models.vit import interpolate_pos_embed
+
+
+class BlipBase(BaseModel):
+
+    @classmethod
+    def init_tokenizer(cls):
+        tokenizer = AutoTokenizer.from_pretrained('AI-ModelScope/bert-base-uncased')
+        tokenizer.add_special_tokens({'bos_token': '[DEC]'})
+        tokenizer.add_special_tokens({'additional_special_tokens': ['[ENC]']})
+        tokenizer.enc_token_id = tokenizer.additional_special_tokens_ids[0]
+        return tokenizer
+
+    def load_from_pretrained(self, url_or_filename):
+        if is_url(url_or_filename):
+            cached_file = download_cached_file(url_or_filename, check_hash=False, progress=True)
+            checkpoint = torch.load(cached_file, map_location='cpu')
+        elif os.path.isfile(url_or_filename):
+            checkpoint = torch.load(url_or_filename, map_location='cpu')
+        else:
+            raise RuntimeError('checkpoint url or path is invalid')
+
+        state_dict = checkpoint['model']
+
+        state_dict['visual_encoder.pos_embed'] = interpolate_pos_embed(
+            state_dict['visual_encoder.pos_embed'], self.visual_encoder
+        )
+        if 'visual_encoder_m.pos_embed' in self.state_dict().keys():
+            state_dict['visual_encoder_m.pos_embed'] = interpolate_pos_embed(
+                state_dict['visual_encoder_m.pos_embed'], self.visual_encoder_m
+            )
+
+        for key in self.state_dict().keys():
+            if key in state_dict.keys():
+                if state_dict[key].shape != self.state_dict()[key].shape:
+                    del state_dict[key]
+
+        msg = self.load_state_dict(state_dict, strict=False)
+
+        logging.info('Missing keys {}'.format(msg.missing_keys))
+        logging.info('load checkpoint from %s' % url_or_filename)
+
+        return msg

evalscope/metrics/t2v_metrics/models/vqascore_models/lavis/models/blip_models/blip_caption.py

@@ -0,0 +1,212 @@
+"""
+ Copyright (c) 2022, salesforce.com, inc.
+ All rights reserved.
+ SPDX-License-Identifier: BSD-3-Clause
+ For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
+"""
+
+import torch
+
+from ...common.registry import registry
+from ..med import XBertLMHeadDecoder
+from ..vit import VisionTransformerEncoder
+from .blip import BlipBase
+from .blip_outputs import BlipIntermediateOutput, BlipOutput
+
+
+@registry.register_model('blip_caption')
+class BlipCaption(BlipBase):
+    """
+    BLIP captioning model.
+
+    Supported model types:
+        - base_coco: fine-tuned BLIP base model on COCO caption dataset (Karparthy split).
+        - large_coco: fine-tuned BLIP large model on COCO caption dataset (Karparthy split).
+
+    Usage:
+        >>> from lavis.models import load_model
+        >>> model = load_model("blip_caption", "base_coco")
+        >>> model = load_model("blip_caption", "large_coco")
+    """
+
+    PRETRAINED_MODEL_CONFIG_DICT = {
+        'base_coco': 'configs/models/blip_caption_base_coco.yaml',
+        'large_coco': 'configs/models/blip_caption_large_coco.yaml',
+    }
+
+    def __init__(self, image_encoder, text_decoder, prompt=None, max_txt_len=40):
+        super().__init__()
+
+        self.tokenizer = self.init_tokenizer()
+
+        self.visual_encoder = image_encoder
+        self.text_decoder = text_decoder
+
+        self.prompt = prompt
+        self.prompt_length = len(self.tokenizer(self.prompt).input_ids) - 1
+
+        self.max_txt_len = max_txt_len
+
+    def forward_encoder(self, samples):
+        image_embeds = self.visual_encoder.forward_features(samples['image'])
+        return image_embeds
+
+    def forward_decoder(self, samples, image_embeds):
+        # prepare inputs for forwarding decoder
+        raw_text = samples['text_input']
+        text = self.tokenizer(
+            raw_text,
+            padding='longest',
+            truncation=True,
+            max_length=self.max_txt_len,
+            return_tensors='pt',
+        ).to(self.device)
+        text.input_ids[:, 0] = self.tokenizer.bos_token_id
+
+        # prepare targets for forwarding decoder
+        decoder_targets = text.input_ids.masked_fill(text.input_ids == self.tokenizer.pad_token_id, -100)
+        decoder_targets[:, :self.prompt_length] = -100
+
+        # forward decoder
+        image_atts = torch.ones(image_embeds.size()[:-1], dtype=torch.long).to(self.device)
+        decoder_output = self.text_decoder(
+            input_ids=text.input_ids,
+            attention_mask=text.attention_mask,
+            encoder_hidden_states=image_embeds,
+            encoder_attention_mask=image_atts,
+            labels=decoder_targets,
+            return_dict=True,
+        )
+
+        return decoder_output, decoder_targets
+
+    def forward(self, samples):
+        r"""
+        Args:
+            samples (dict): A dictionary containing the following keys:
+                - image (torch.Tensor): A tensor of shape (batch_size, 3, H, W)
+                - text_input (list): A list of strings of length batch_size.
+        Returns:
+            output (BlipOutput): A BlipOutput object containing the following
+                attributes:
+                - loss (torch.Tensor): A scalar tensor containing the total loss. For BlipCaption, this is the same as the LM loss.
+                - loss_lm (torch.Tensor): A scalar tensor containing the LM loss.
+                - intermediate_outputs (BlipIntermediateOutput): A BlipIntermediateOutput object containing intermediate outputs.
+                  see :class:`lavis.models.blip_models.blip_outputs.BlipOutput` for more details.
+
+        Example:
+        ```python
+        >>> from PIL import Image
+        >>> from lavis.models import load_model_and_preprocess
+        >>> model, vis_processors, txt_processors = load_model_and_preprocess("blip_caption")
+        >>> raw_image = Image.open("docs/data/merlion.png").convert("RGB")
+        >>> image = vis_processors["eval"](raw_image).unsqueeze(0)
+        >>> text_input = ["a large statue of a person spraying water from a fountain"]
+        >>> samples = {"image": image, "text_input": text_input}
+        >>> output = model(samples)
+        >>> output.keys()
+        odict_keys(['intermediate_output', 'loss', 'loss_lm'])
+        >>> output.intermediate_output.image_embeds.shape
+        torch.Size([1, 577, 768])
+        >>> output.intermediate_output.decoder_labels.shape
+        torch.Size([1, 13])
+        ```"""
+
+        image_embeds = self.forward_encoder(samples)
+        decoder_output, decoder_targets = self.forward_decoder(samples, image_embeds)
+
+        # return decoder_out
+        return BlipOutput(
+            loss=decoder_output.loss,
+            loss_lm=decoder_output.loss,
+            intermediate_output=BlipIntermediateOutput(
+                image_embeds=image_embeds,
+                decoder_output=decoder_output,
+                decoder_labels=decoder_targets,
+            ),
+        )
+
+    def generate(
+        self,
+        samples,
+        use_nucleus_sampling=False,
+        num_beams=3,
+        max_length=30,
+        min_length=10,
+        top_p=0.9,
+        repetition_penalty=1.0,
+        num_captions=1,
+    ):
+        """
+        Args:
+            samples (dict): A dictionary containing the following keys:
+                - image (torch.Tensor): A tensor of shape (batch_size, 3, H, W)
+            use_nucleus_sampling (bool): Whether to use nucleus sampling. If False, use top-k sampling.
+            num_beams (int): Number of beams for beam search. 1 means no beam search.
+            max_length (int): The maximum length of the sequence to be generated.
+            min_length (int): The minimum length of the sequence to be generated.
+            top_p (float): The cumulative probability for nucleus sampling.
+            repetition_penalty (float): The parameter for repetition penalty. 1.0 means no penalty.
+            num_captions (int): Number of captions to be generated for each image.
+        Returns:
+            captions (list): A list of strings of length batch_size * num_captions.
+
+        Example:
+        ```python
+        >>> from PIL import Image
+        >>> from lavis.models import load_model_and_preprocess
+        >>> model, vis_processors, txt_processors = load_model_and_preprocess("blip_caption")
+        >>> raw_image = Image.open("docs/data/merlion.png").convert("RGB")
+        >>> image = vis_processors["eval"](raw_image).unsqueeze(0)
+        >>> samples = {"image": image}
+        >>> captions = model.generate(samples)
+        >>> captions
+        ['a large statue of a person spraying water from a fountain']
+        >>> captions = model.generate(samples, use_nucleus_sampling=True, num_captions=3)
+        >>> captions # example output, results may vary due to randomness
+        ['singapore showing the view of some building',
+        'the singapore harbor in twilight, as the weather is going down',
+        'the famous singapore fountain at sunset']
+        """
+        # prepare inputs for decoder generation.
+        encoder_out = self.forward_encoder(samples)
+        image_embeds = torch.repeat_interleave(encoder_out, num_captions, 0)
+
+        prompt = [self.prompt] * image_embeds.size(0)
+        prompt = self.tokenizer(prompt, return_tensors='pt').to(self.device)
+        prompt.input_ids[:, 0] = self.tokenizer.bos_token_id
+        prompt.input_ids = prompt.input_ids[:, :-1]
+
+        # get decoded text
+        decoder_out = self.text_decoder.generate_from_encoder(
+            tokenized_prompt=prompt,
+            visual_embeds=image_embeds,
+            sep_token_id=self.tokenizer.sep_token_id,
+            pad_token_id=self.tokenizer.pad_token_id,
+            use_nucleus_sampling=use_nucleus_sampling,
+            num_beams=num_beams,
+            max_length=max_length,
+            min_length=min_length,
+            top_p=top_p,
+            repetition_penalty=repetition_penalty,
+        )
+
+        outputs = self.tokenizer.batch_decode(decoder_out, skip_special_tokens=True)
+        captions = [output[len(self.prompt):] for output in outputs]
+
+        return captions
+
+    @classmethod
+    def from_config(cls, cfg):
+        # vision encoder
+        image_encoder = VisionTransformerEncoder.from_config(cfg)
+        # text encoder + multimodal decoder
+        text_decoder = XBertLMHeadDecoder.from_config(cfg)
+
+        prompt = cfg.get('prompt', None)
+        max_txt_len = cfg.get('max_txt_len', 40)
+
+        model = cls(image_encoder, text_decoder, prompt=prompt, max_txt_len=max_txt_len)
+        model.load_checkpoint_from_config(cfg)
+
+        return model

evalscope/metrics/t2v_metrics/models/vqascore_models/lavis/models/blip_models/blip_classification.py

@@ -0,0 +1,164 @@
+"""
+ Copyright (c) 2022, salesforce.com, inc.
+ All rights reserved.
+ SPDX-License-Identifier: BSD-3-Clause
+ For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
+"""
+
+import torch
+import torch.nn.functional as F
+from copy import deepcopy
+from torch import nn
+
+from ...common.registry import registry
+from ..base_model import MomentumDistilationMixin
+from ..med import XBertEncoder
+from ..vit import VisionTransformerEncoder
+from .blip import BlipBase
+from .blip_outputs import BlipIntermediateOutput, BlipOutputWithLogits
+
+
+@registry.register_model('blip_classification')
+class BlipClassification(BlipBase, MomentumDistilationMixin):
+    PRETRAINED_MODEL_CONFIG_DICT = {
+        'base': 'configs/models/blip_classification_base.yaml',
+    }
+
+    def __init__(
+        self,
+        image_encoder,
+        text_encoder,
+        num_classes,
+        momentum=0.995,
+        alpha=0.4,
+        max_txt_len=40,
+        use_distill=True,
+    ):
+        super().__init__()
+
+        self.tokenizer = self.init_tokenizer()
+
+        self.use_distill = use_distill
+
+        self.visual_encoder = image_encoder
+        self.text_encoder = text_encoder
+
+        hidden_size = text_encoder.config.hidden_size
+        self.cls_head = nn.Sequential(
+            nn.Linear(hidden_size, hidden_size),
+            nn.ReLU(),
+            nn.Linear(hidden_size, num_classes),
+        )
+
+        if self.use_distill:
+            self.visual_encoder_m = deepcopy(self.visual_encoder)
+            self.text_encoder_m = deepcopy(self.text_encoder)
+            self.cls_head_m = deepcopy(self.cls_head)
+
+            self.momentum = momentum
+            self.alpha = alpha
+
+            self.model_pairs = [
+                [self.visual_encoder, self.visual_encoder_m],
+                [self.text_encoder, self.text_encoder_m],
+                [self.cls_head, self.cls_head_m],
+            ]
+
+            self.copy_params()
+
+        self.max_txt_len = max_txt_len
+
+    def _rampup_factor(self, epoch, iters, num_iters_per_epoch):
+        return min(1, (epoch * num_iters_per_epoch + iters) / num_iters_per_epoch)
+
+    def forward(self, samples, is_train=True):
+        sentences = samples['text_input']
+        sentences = self.tokenizer(
+            sentences,
+            padding='longest',
+            truncation=True,
+            max_length=self.max_txt_len,
+            return_tensors='pt',
+        ).to(self.device)
+        samples.update({'tokenized_text': sentences})
+
+        targets = samples['label']
+
+        image_embeds = self.visual_encoder.forward_features(samples['image'])
+        encoder_output = self.text_encoder.forward_automask(samples['tokenized_text'], image_embeds)
+
+        prediction = self.cls_head(encoder_output.last_hidden_state[:, 0, :])
+
+        if is_train:
+            if self.use_distill:
+                with torch.no_grad():
+                    self._momentum_update()
+
+                    image_embeds_m = self.visual_encoder_m(samples['image'])
+                    encoder_output_m = self.text_encoder_m.forward_automask(samples['tokenized_text'], image_embeds_m)
+
+                    prediction_m = self.cls_head_m(encoder_output_m.last_hidden_state[:, 0, :])
+
+                alpha = self.alpha * self._rampup_factor(
+                    epoch=samples['epoch'],
+                    iters=samples['iters'],
+                    num_iters_per_epoch=samples['num_iters_per_epoch'],
+                )
+
+                loss = (1 - alpha) * F.cross_entropy(prediction, targets) - alpha * torch.sum(
+                    F.log_softmax(prediction, dim=1) * F.softmax(prediction_m, dim=1),
+                    dim=1,
+                ).mean()
+            else:
+                loss = F.cross_entropy(prediction, targets)
+
+            # return {"loss": loss}
+            return BlipOutputWithLogits(
+                loss=loss,
+                intermediate_output=BlipIntermediateOutput(
+                    image_embeds=image_embeds,
+                    image_embeds_m=image_embeds_m,
+                    encoder_output=encoder_output,
+                    encoder_output_m=encoder_output_m,
+                ),
+                logits=prediction,
+                logits_m=prediction_m,
+            )
+
+        else:
+            return {'predictions': prediction, 'targets': targets}
+
+    def predict(self, samples):
+        output = self.forward(samples, is_train=False)
+        return output
+
+    @classmethod
+    def from_config(cls, cfg=None):
+        image_encoder = VisionTransformerEncoder.from_config(cfg)
+
+        # text encoder + multimodal encoder
+        text_encoder = XBertEncoder.from_config(cfg)
+        use_distill = cfg.get('use_distill', True)
+        momentum = cfg.get('momentum', 0.995)
+        num_classes = cfg.get('num_classes', -1)
+        alpha = cfg.get('alpha', 0.4)
+        max_txt_len = cfg.get('max_txt_len', 40)
+
+        assert num_classes > 1, 'Invalid number of classes provided, found {}'.format(num_classes)
+
+        model = cls(
+            image_encoder=image_encoder,
+            text_encoder=text_encoder,
+            use_distill=use_distill,
+            alpha=alpha,
+            num_classes=num_classes,
+            momentum=momentum,
+            max_txt_len=max_txt_len,
+        )
+
+        # load pre-trained weights
+        pretrain_path = cfg.get('pretrained', None)
+        if pretrain_path is not None:
+            msg = model.load_from_pretrained(url_or_filename=pretrain_path)
+
+        return model