evalscope 0.10.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

evalscope/metrics/t2v_metrics/models/clipscore_models/build_mps_model/cross_modeling.py

@@ -0,0 +1,282 @@
+import torch
+import torch.nn.functional as F
+from einops import rearrange, repeat
+from torch import einsum, nn
+
+# helper functions
+
+
+def exists(val):
+    return val is not None
+
+
+def default(val, d):
+    return val if exists(val) else d
+
+
+# normalization
+# they use layernorm without bias, something that pytorch does not offer
+
+
+class LayerNorm(nn.Module):
+
+    def __init__(self, dim):
+        super().__init__()
+        self.weight = nn.Parameter(torch.ones(dim))
+        self.register_buffer('bias', torch.zeros(dim))
+
+    def forward(self, x):
+        return F.layer_norm(x, x.shape[-1:], self.weight, self.bias)
+
+
+# residual
+
+
+class Residual(nn.Module):
+
+    def __init__(self, fn):
+        super().__init__()
+        self.fn = fn
+
+    def forward(self, x, *args, **kwargs):
+        return self.fn(x, *args, **kwargs) + x
+
+
+# rotary positional embedding
+# https://arxiv.org/abs/2104.09864
+
+
+class RotaryEmbedding(nn.Module):
+
+    def __init__(self, dim):
+        super().__init__()
+        inv_freq = 1.0 / (10000**(torch.arange(0, dim, 2).float() / dim))
+        self.register_buffer('inv_freq', inv_freq)
+
+    def forward(self, max_seq_len, *, device):
+        seq = torch.arange(max_seq_len, device=device, dtype=self.inv_freq.dtype)
+        freqs = einsum('i , j -> i j', seq, self.inv_freq)
+        return torch.cat((freqs, freqs), dim=-1)
+
+
+def rotate_half(x):
+    x = rearrange(x, '... (j d) -> ... j d', j=2)
+    x1, x2 = x.unbind(dim=-2)
+    return torch.cat((-x2, x1), dim=-1)
+
+
+def apply_rotary_pos_emb(pos, t):
+    return (t * pos.cos()) + (rotate_half(t) * pos.sin())
+
+
+# classic Noam Shazeer paper, except here they use SwiGLU instead of the more popular GEGLU for gating the feedforward
+# https://arxiv.org/abs/2002.05202
+
+
+class SwiGLU(nn.Module):
+
+    def forward(self, x):
+        x, gate = x.chunk(2, dim=-1)
+        return F.silu(gate) * x
+
+
+# parallel attention and feedforward with residual
+# discovered by Wang et al + EleutherAI from GPT-J fame
+
+
+class ParallelTransformerBlock(nn.Module):
+
+    def __init__(self, dim, dim_head=64, heads=8, ff_mult=4):
+        super().__init__()
+        self.norm = LayerNorm(dim)
+
+        attn_inner_dim = dim_head * heads
+        ff_inner_dim = dim * ff_mult
+        self.fused_dims = (attn_inner_dim, dim_head, dim_head, (ff_inner_dim * 2))
+
+        self.heads = heads
+        self.scale = dim_head**-0.5
+        self.rotary_emb = RotaryEmbedding(dim_head)
+
+        self.fused_attn_ff_proj = nn.Linear(dim, sum(self.fused_dims), bias=False)
+        self.attn_out = nn.Linear(attn_inner_dim, dim, bias=False)
+
+        self.ff_out = nn.Sequential(SwiGLU(), nn.Linear(ff_inner_dim, dim, bias=False))
+
+        self.register_buffer('pos_emb', None, persistent=False)
+
+    def get_rotary_embedding(self, n, device):
+        if self.pos_emb is not None and self.pos_emb.shape[-2] >= n:
+            return self.pos_emb[:n]
+
+        pos_emb = self.rotary_emb(n, device=device)
+        self.register_buffer('pos_emb', pos_emb, persistent=False)
+        return pos_emb
+
+    def forward(self, x, attn_mask=None):
+        """
+        einstein notation
+        b - batch
+        h - heads
+        n, i, j - sequence length (base sequence length, source, target)
+        d - feature dimension
+        """
+
+        n, device, h = x.shape[1], x.device, self.heads
+
+        # pre layernorm
+
+        x = self.norm(x)
+
+        # attention queries, keys, values, and feedforward inner
+
+        q, k, v, ff = self.fused_attn_ff_proj(x).split(self.fused_dims, dim=-1)
+
+        # split heads
+        # they use multi-query single-key-value attention, yet another Noam Shazeer paper
+        # they found no performance loss past a certain scale, and more efficient decoding obviously
+        # https://arxiv.org/abs/1911.02150
+
+        q = rearrange(q, 'b n (h d) -> b h n d', h=h)
+
+        # rotary embeddings
+
+        positions = self.get_rotary_embedding(n, device)
+        q, k = map(lambda t: apply_rotary_pos_emb(positions, t), (q, k))
+
+        # scale
+
+        q = q * self.scale
+
+        # similarity
+
+        sim = einsum('b h i d, b j d -> b h i j', q, k)
+
+        # extra attention mask - for masking out attention from text CLS token to padding
+
+        if exists(attn_mask):
+            attn_mask = rearrange(attn_mask, 'b i j -> b 1 i j')
+            sim = sim.masked_fill(~attn_mask, -torch.finfo(sim.dtype).max)
+
+        # attention
+
+        sim = sim - sim.amax(dim=-1, keepdim=True).detach()
+        attn = sim.softmax(dim=-1)
+
+        # aggregate values
+
+        out = einsum('b h i j, b j d -> b h i d', attn, v)
+
+        # merge heads
+
+        out = rearrange(out, 'b h n d -> b n (h d)')
+        return self.attn_out(out) + self.ff_out(ff)
+
+
+# cross attention - using multi-query + one-headed key / values as in PaLM w/ optional parallel feedforward
+
+
+class CrossAttention(nn.Module):
+
+    def __init__(
+        self, dim, *, context_dim=None, dim_head=64, heads=12, parallel_ff=False, ff_mult=4, norm_context=False
+    ):
+        super().__init__()
+        self.heads = heads
+        self.scale = dim_head**-0.5
+        inner_dim = heads * dim_head
+        context_dim = default(context_dim, dim)
+
+        self.norm = LayerNorm(dim)
+        self.context_norm = LayerNorm(context_dim) if norm_context else nn.Identity()
+
+        self.to_q = nn.Linear(dim, inner_dim, bias=False)
+        self.to_kv = nn.Linear(context_dim, dim_head * 2, bias=False)
+        self.to_out = nn.Linear(inner_dim, dim, bias=False)
+
+        # whether to have parallel feedforward
+
+        ff_inner_dim = ff_mult * dim
+
+        self.ff = nn.Sequential(
+            nn.Linear(dim, ff_inner_dim * 2, bias=False), SwiGLU(), nn.Linear(ff_inner_dim, dim, bias=False)
+        ) if parallel_ff else None
+
+    def forward(self, x, context, mask):
+        """
+        einstein notation
+        b - batch
+        h - heads
+        n, i, j - sequence length (base sequence length, source, target)
+        d - feature dimension
+        """
+
+        # pre-layernorm, for queries and context
+
+        x = self.norm(x)
+        context = self.context_norm(context)
+
+        # get queries
+
+        q = self.to_q(x)
+        q = rearrange(q, 'b n (h d) -> b h n d', h=self.heads)
+
+        # scale
+
+        q = q * self.scale
+
+        # get key / values
+
+        k, v = self.to_kv(context).chunk(2, dim=-1)
+
+        # query / key similarity
+
+        sim = einsum('b h i d, b j d -> b h i j', q, k)
+
+        # attention
+        mask = mask.unsqueeze(1).repeat(1, self.heads, 1, 1)
+        sim = sim + mask  # context mask
+        sim = sim - sim.amax(dim=-1, keepdim=True)
+        attn = sim.softmax(dim=-1)
+
+        # aggregate
+
+        out = einsum('b h i j, b j d -> b h i d', attn, v)
+
+        # merge and combine heads
+
+        out = rearrange(out, 'b h n d -> b n (h d)')
+        out = self.to_out(out)
+
+        # add parallel feedforward (for multimodal layers)
+
+        if exists(self.ff):
+            out = out + self.ff(x)
+
+        return out
+
+
+class Cross_model(nn.Module):
+
+    def __init__(self, dim=512, layer_num=4, dim_head=64, heads=8, ff_mult=4):
+        super().__init__()
+
+        self.layers = nn.ModuleList([])
+
+        for ind in range(layer_num):
+            self.layers.append(
+                nn.ModuleList([
+                    Residual(
+                        CrossAttention(dim=dim, dim_head=dim_head, heads=heads, parallel_ff=True, ff_mult=ff_mult)
+                    ),
+                    Residual(ParallelTransformerBlock(dim=dim, dim_head=dim_head, heads=heads, ff_mult=ff_mult))
+                ])
+            )
+
+    def forward(self, query_tokens, context_tokens, mask):
+
+        for cross_attn, self_attn_ff in self.layers:
+            query_tokens = cross_attn(query_tokens, context_tokens, mask)
+            query_tokens = self_attn_ff(query_tokens)
+
+        return query_tokens

evalscope/metrics/t2v_metrics/models/clipscore_models/clip_model.py

@@ -0,0 +1,115 @@
+import os
+import torch
+from typing import List
+
+from ...constants import CACHE_DIR
+from ..model import ScoreModel
+
+CLIP_MODELS = [
+    'openai:RN50', 'yfcc15m:RN50', 'cc12m:RN50', 'openai:RN101', 'yfcc15m:RN101', 'openai:RN50x4', 'openai:RN50x16',
+    'openai:RN50x64', 'openai:ViT-B-32', 'laion400m_e31:ViT-B-32', 'laion400m_e32:ViT-B-32', 'laion2b_e16:ViT-B-32',
+    'laion2b_s34b_b79k:ViT-B-32', 'datacomp_xl_s13b_b90k:ViT-B-32', 'datacomp_m_s128m_b4k:ViT-B-32',
+    'commonpool_m_clip_s128m_b4k:ViT-B-32', 'commonpool_m_laion_s128m_b4k:ViT-B-32',
+    'commonpool_m_image_s128m_b4k:ViT-B-32', 'commonpool_m_text_s128m_b4k:ViT-B-32',
+    'commonpool_m_basic_s128m_b4k:ViT-B-32', 'commonpool_m_s128m_b4k:ViT-B-32', 'datacomp_s_s13m_b4k:ViT-B-32',
+    'commonpool_s_clip_s13m_b4k:ViT-B-32', 'commonpool_s_laion_s13m_b4k:ViT-B-32',
+    'commonpool_s_image_s13m_b4k:ViT-B-32', 'commonpool_s_text_s13m_b4k:ViT-B-32',
+    'commonpool_s_basic_s13m_b4k:ViT-B-32', 'commonpool_s_s13m_b4k:ViT-B-32', 'metaclip_400m:ViT-B-32',
+    'metaclip_fullcc:ViT-B-32', 'datacomp_s34b_b86k:ViT-B-32-256', 'openai:ViT-B-16', 'laion400m_e31:ViT-B-16',
+    'laion400m_e32:ViT-B-16', 'laion2b_s34b_b88k:ViT-B-16', 'datacomp_xl_s13b_b90k:ViT-B-16',
+    'datacomp_l_s1b_b8k:ViT-B-16', 'commonpool_l_clip_s1b_b8k:ViT-B-16', 'commonpool_l_laion_s1b_b8k:ViT-B-16',
+    'commonpool_l_image_s1b_b8k:ViT-B-16', 'commonpool_l_text_s1b_b8k:ViT-B-16', 'commonpool_l_basic_s1b_b8k:ViT-B-16',
+    'commonpool_l_s1b_b8k:ViT-B-16', 'dfn2b:ViT-B-16', 'metaclip_400m:ViT-B-16', 'metaclip_fullcc:ViT-B-16',
+    'laion400m_e31:ViT-B-16-plus-240', 'laion400m_e32:ViT-B-16-plus-240', 'openai:ViT-L-14', 'laion400m_e31:ViT-L-14',
+    'laion400m_e32:ViT-L-14', 'laion2b_s32b_b82k:ViT-L-14', 'datacomp_xl_s13b_b90k:ViT-L-14',
+    'commonpool_xl_clip_s13b_b90k:ViT-L-14', 'commonpool_xl_laion_s13b_b90k:ViT-L-14',
+    'commonpool_xl_s13b_b90k:ViT-L-14', 'metaclip_400m:ViT-L-14', 'metaclip_fullcc:ViT-L-14', 'dfn2b:ViT-L-14',
+    'dfn2b_s39b:ViT-L-14', 'openai:ViT-L-14-336', 'laion2b_s32b_b79k:ViT-H-14', 'metaclip_fullcc:ViT-H-14',
+    'metaclip_altogether:ViT-H-14', 'dfn5b:ViT-H-14', 'dfn5b:ViT-H-14-378', 'laion2b_s12b_b42k:ViT-g-14',
+    'laion2b_s34b_b88k:ViT-g-14', 'laion2b_s39b_b160k:ViT-bigG-14', 'metaclip_fullcc:ViT-bigG-14',
+    'laion2b_s12b_b32k:roberta-ViT-B-32', 'laion5b_s13b_b90k:xlm-roberta-base-ViT-B-32',
+    'frozen_laion5b_s13b_b90k:xlm-roberta-large-ViT-H-14', 'laion400m_s13b_b51k:convnext_base',
+    'laion2b_s13b_b82k:convnext_base_w', 'laion2b_s13b_b82k_augreg:convnext_base_w',
+    'laion_aesthetic_s13b_b82k:convnext_base_w', 'laion_aesthetic_s13b_b82k:convnext_base_w_320',
+    'laion_aesthetic_s13b_b82k_augreg:convnext_base_w_320', 'laion2b_s26b_b102k_augreg:convnext_large_d',
+    'laion2b_s29b_b131k_ft:convnext_large_d_320', 'laion2b_s29b_b131k_ft_soup:convnext_large_d_320',
+    'laion2b_s34b_b82k_augreg:convnext_xxlarge', 'laion2b_s34b_b82k_augreg_rewind:convnext_xxlarge',
+    'laion2b_s34b_b82k_augreg_soup:convnext_xxlarge', 'laion2b_s13b_b90k:coca_ViT-B-32',
+    'mscoco_finetuned_laion2b_s13b_b90k:coca_ViT-B-32', 'laion2b_s13b_b90k:coca_ViT-L-14',
+    'mscoco_finetuned_laion2b_s13b_b90k:coca_ViT-L-14', 'laion400m_s11b_b41k:EVA01-g-14',
+    'merged2b_s11b_b114k:EVA01-g-14-plus', 'merged2b_s8b_b131k:EVA02-B-16', 'merged2b_s4b_b131k:EVA02-L-14',
+    'merged2b_s6b_b61k:EVA02-L-14-336', 'laion2b_s4b_b115k:EVA02-E-14', 'laion2b_s9b_b144k:EVA02-E-14-plus',
+    'webli:ViT-B-16-SigLIP', 'webli:ViT-B-16-SigLIP-256', 'webli:ViT-B-16-SigLIP-i18n-256', 'webli:ViT-B-16-SigLIP-384',
+    'webli:ViT-B-16-SigLIP-512', 'webli:ViT-L-16-SigLIP-256', 'webli:ViT-L-16-SigLIP-384', 'webli:ViT-SO400M-14-SigLIP',
+    'webli:ViT-SO400M-16-SigLIP-i18n-256', 'webli:ViT-SO400M-14-SigLIP-378', 'webli:ViT-SO400M-14-SigLIP-384',
+    'webli:ViT-B-32-SigLIP2-256', 'webli:ViT-B-16-SigLIP2', 'webli:ViT-B-16-SigLIP2-256', 'webli:ViT-B-16-SigLIP2-384',
+    'webli:ViT-B-16-SigLIP2-512', 'webli:ViT-L-16-SigLIP2-256', 'webli:ViT-L-16-SigLIP2-384',
+    'webli:ViT-L-16-SigLIP2-512', 'webli:ViT-SO400M-14-SigLIP2', 'webli:ViT-SO400M-14-SigLIP2-378',
+    'webli:ViT-SO400M-16-SigLIP2-256', 'webli:ViT-SO400M-16-SigLIP2-384', 'webli:ViT-SO400M-16-SigLIP2-512',
+    'webli:ViT-gopt-16-SigLIP2-256', 'webli:ViT-gopt-16-SigLIP2-384', 'datacomp1b:ViT-L-14-CLIPA',
+    'datacomp1b:ViT-L-14-CLIPA-336', 'datacomp1b:ViT-H-14-CLIPA', 'laion2b:ViT-H-14-CLIPA-336',
+    'datacomp1b:ViT-H-14-CLIPA-336', 'datacomp1b:ViT-bigG-14-CLIPA', 'datacomp1b:ViT-bigG-14-CLIPA-336',
+    'v1:nllb-clip-base', 'v1:nllb-clip-large', 'v1:nllb-clip-base-siglip', 'mrl:nllb-clip-base-siglip',
+    'v1:nllb-clip-large-siglip', 'mrl:nllb-clip-large-siglip', 'datacompdr:MobileCLIP-S1', 'datacompdr:MobileCLIP-S2',
+    'datacompdr:MobileCLIP-B', 'datacompdr_lt:MobileCLIP-B', 'datacomp1b:ViTamin-S', 'datacomp1b:ViTamin-S-LTT',
+    'datacomp1b:ViTamin-B', 'datacomp1b:ViTamin-B-LTT', 'datacomp1b:ViTamin-L', 'datacomp1b:ViTamin-L-256',
+    'datacomp1b:ViTamin-L-336', 'datacomp1b:ViTamin-L-384', 'datacomp1b:ViTamin-L2', 'datacomp1b:ViTamin-L2-256',
+    'datacomp1b:ViTamin-L2-336', 'datacomp1b:ViTamin-L2-384', 'datacomp1b:ViTamin-XL-256', 'datacomp1b:ViTamin-XL-336',
+    'datacomp1b:ViTamin-XL-384', 'openai:RN50-quickgelu', 'yfcc15m:RN50-quickgelu', 'cc12m:RN50-quickgelu',
+    'openai:RN101-quickgelu', 'yfcc15m:RN101-quickgelu', 'openai:RN50x4-quickgelu', 'openai:RN50x16-quickgelu',
+    'openai:RN50x64-quickgelu', 'openai:ViT-B-32-quickgelu', 'laion400m_e31:ViT-B-32-quickgelu',
+    'laion400m_e32:ViT-B-32-quickgelu', 'metaclip_400m:ViT-B-32-quickgelu', 'metaclip_fullcc:ViT-B-32-quickgelu',
+    'openai:ViT-B-16-quickgelu', 'dfn2b:ViT-B-16-quickgelu', 'metaclip_400m:ViT-B-16-quickgelu',
+    'metaclip_fullcc:ViT-B-16-quickgelu', 'openai:ViT-L-14-quickgelu', 'metaclip_400m:ViT-L-14-quickgelu',
+    'metaclip_fullcc:ViT-L-14-quickgelu', 'dfn2b:ViT-L-14-quickgelu', 'openai:ViT-L-14-336-quickgelu',
+    'metaclip_fullcc:ViT-H-14-quickgelu', 'dfn5b:ViT-H-14-quickgelu', 'dfn5b:ViT-H-14-378-quickgelu',
+    'metaclip_fullcc:ViT-bigG-14-quickgelu'
+]  # noqa: E501
+
+
+class CLIPScoreModel(ScoreModel):
+    "A wrapper for OpenCLIP models (including openAI's CLIP, OpenCLIP, DatacompCLIP)"
+
+    def __init__(self, model_name='openai:ViT-L-14', device='cuda', cache_dir=CACHE_DIR):
+        assert model_name in CLIP_MODELS
+        super().__init__(model_name=model_name, device=device, cache_dir=cache_dir)
+
+    def load_model(self):
+        """Load the model, tokenizer, image transform
+        """
+        import open_clip
+
+        from ..utils import download_open_clip_model
+
+        self.pretrained, self.arch = self.model_name.split(':')
+        # load model from modelscope
+        model_file_path = download_open_clip_model(self.arch, self.pretrained, self.cache_dir)
+
+        self.model, _, self.preprocess = open_clip.create_model_and_transforms(
+            self.arch, pretrained=model_file_path, device=self.device
+        )
+        self.tokenizer = open_clip.get_tokenizer(self.arch)
+        self.model.eval()
+
+    def load_images(self, image: List[str]) -> torch.Tensor:
+        """Load the image(s), and return a tensor (after preprocessing) put on self.device
+        """
+        image = [self.image_loader(x) for x in image]
+        image = [self.preprocess(x) for x in image]
+        image = torch.stack(image, dim=0).to(self.device)
+        return image
+
+    @torch.no_grad()
+    def forward(self, images: List[str], texts: List[str]) -> torch.Tensor:
+        """Forward pass of the model to return n scores for n (image, text) pairs (in PyTorch Tensor)
+        """
+        assert len(images) == len(texts)
+        image = self.load_images(images)
+        text = self.tokenizer(texts).to(self.device)
+        image_features = self.model.encode_image(image)
+        image_features /= image_features.norm(dim=-1, keepdim=True)
+        text_features = self.model.encode_text(text)
+        text_features /= text_features.norm(dim=-1, keepdim=True)
+
+        # return cosine similarity as scores
+        return (image_features * text_features).sum(dim=-1)

evalscope/metrics/t2v_metrics/models/clipscore_models/hpsv2_model.py

@@ -0,0 +1,87 @@
+import torch
+from typing import List
+
+from ...constants import CACHE_DIR
+from ..model import ScoreModel
+
+HPSV2_MODELS = ['hpsv2', 'hpsv2.1']
+HPS_VERSION_MAP = {
+    'hpsv2': 'HPS_v2_compressed.pt',
+    'hpsv2.1': 'HPS_v2.1_compressed.pt',
+}
+
+
+class HPSV2ScoreModel(ScoreModel):
+    'A wrapper for HPSv2 models '
+
+    def __init__(self, model_name='openai:ViT-L-14', device='cuda', cache_dir=CACHE_DIR):
+        assert model_name in HPSV2_MODELS
+        super().__init__(model_name=model_name, device=device, cache_dir=cache_dir)
+
+    def load_model(self):
+        """Load the model, tokenizer, image transform
+        """
+        import open_clip
+
+        from ..utils import download_file, download_open_clip_model
+
+        self.pretrained, self.arch = 'laion2B-s32B-b79K:ViT-H-14'.split(':')
+        # load model from modelscope
+        model_file_path = download_open_clip_model(self.arch, self.pretrained, self.cache_dir)
+
+        self.model, _, self.preprocess = open_clip.create_model_and_transforms(
+            self.arch,
+            pretrained=model_file_path,
+            precision='amp',
+            device=self.device,
+            jit=False,
+            force_quick_gelu=False,
+            force_custom_text=False,
+            force_patch_dropout=False,
+            force_image_size=None,
+            pretrained_image=False,
+            image_mean=None,
+            image_std=None,
+            image_resize_mode='longest',
+            aug_cfg={},
+            output_dict=True
+        )
+
+        # update weight
+        model_weight_path = download_file('AI-ModelScope/HPSv2', HPS_VERSION_MAP[self.model_name], self.cache_dir)
+        checkpoint = torch.load(model_weight_path, map_location=self.device, weights_only=False)
+        self.model.load_state_dict(checkpoint['state_dict'])
+        self.tokenizer = open_clip.get_tokenizer(self.arch)
+        self.model.eval()
+
+    def load_images(self, image: List[str]):
+        """Load the image(s), and return a tensor (after preprocessing) put on self.device
+        """
+        images = [self.image_loader(x) for x in image]
+        return images
+
+    @torch.no_grad()
+    def forward(self, images: List[str], texts: List[str]) -> torch.Tensor:
+        """Forward pass of the model to return n scores for n (image, text) pairs (in PyTorch Tensor)
+        """
+        assert len(images) == len(texts)
+        images = self.load_images(images)
+        scores = torch.zeros(len(images), dtype=torch.float16).to(self.device)
+        for i in range(len(images)):
+            caption = texts[i]
+            image = images[i]
+            # Process the image
+            image = self.preprocess(image).unsqueeze(0).to(device=self.device, non_blocking=True)
+            # Process the prompt
+            text = self.tokenizer([caption]).to(device=self.device, non_blocking=True)  # Updated to use texts[i]
+            # Calculate the HPS
+            with torch.amp.autocast(device_type=self.device):
+                outputs = self.model(image, text)
+                image_features, text_features = outputs['image_features'], outputs['text_features']
+                logits_per_image = image_features @ text_features.T
+
+                hps_score = torch.diagonal(logits_per_image).cpu().numpy()
+            scores[i] = float(hps_score[0])
+
+        # return cosine similarity as scores
+        return scores

evalscope/metrics/t2v_metrics/models/clipscore_models/mps_model.py

@@ -0,0 +1,86 @@
+import torch
+from modelscope import AutoProcessor
+from transformers import CLIPConfig
+from typing import List
+
+from ...constants import CACHE_DIR
+from ..model import ScoreModel
+
+MPS_MODELS = ['mps']
+
+
+class MPSModel(ScoreModel):
+    'A wrapper for MPS Score models'
+
+    def __init__(self, model_name='mps', device='cuda', cache_dir=CACHE_DIR):
+        assert model_name in MPS_MODELS
+        super().__init__(model_name=model_name, device=device, cache_dir=cache_dir)
+
+    def load_model(self):
+        """Load the model, tokenizer, image transform
+        """
+        from ..utils import download_file
+        from .build_mps_model.clip_model import CLIPModel
+
+        assert self.model_name == 'mps'
+
+        processor_name_or_path = 'laion/CLIP-ViT-H-14-laion2B-s32B-b79K'
+        self.processor = AutoProcessor.from_pretrained(processor_name_or_path)
+
+        config = download_file('AI-ModelScope/MPS', file_name='config.json', cache_dir=self.cache_dir)
+        model_pretrained_path = download_file(
+            'AI-ModelScope/MPS', file_name='MPS_overall_state_dict.pt', cache_dir=self.cache_dir
+        )  # modelscope model
+        model_weight = torch.load(model_pretrained_path, weights_only=True, map_location='cpu')
+
+        self.model = CLIPModel(config=CLIPConfig.from_json_file(config))
+        self.model.load_state_dict(model_weight, strict=False)
+        self.model.eval().to(self.device)
+
+    def load_images(self, image: List[str]) -> torch.Tensor:
+        """Load the image(s), and return a tensor (no preprocessing!!) put on self.device
+        """
+        image = [self.image_loader(x) for x in image]
+        image = self.processor(images=image, return_tensors='pt')['pixel_values']
+        return image
+
+    def process_text(self, text: List[str]) -> dict:
+        """Process the text(s), and return a tensor (after preprocessing) put on self.device
+        """
+        text_inputs = self.processor(
+            text=text,
+            padding='max_length',
+            truncation=True,
+            return_tensors='pt',
+        ).input_ids
+        return text_inputs
+
+    @torch.no_grad()
+    def forward(self, images: List[str], texts: List[str], condition=None) -> torch.Tensor:
+        """Forward pass of the model to return n scores for n (image, text) pairs (in PyTorch Tensor)
+        """
+        assert len(images) == len(texts)
+        image_input = self.load_images(images).to(self.device)
+        text_input = self.process_text(texts).to(self.device)
+        if condition is None:
+            condition = 'light, color, clarity, tone, style, ambiance, artistry, shape, face, hair, hands, limbs, structure, instance, texture, quantity, attributes, position, number, location, word, things.'
+        condition_batch = self.process_text(condition).repeat(text_input.shape[0], 1).to(self.device)
+
+        # embed
+        text_f, text_features = self.model.model.get_text_features(text_input)
+
+        image_f = self.model.model.get_image_features(image_input.half())
+        condition_f, _ = self.model.model.get_text_features(condition_batch)
+
+        sim_text_condition = torch.einsum('b i d, b j d -> b j i', text_f, condition_f)
+        sim_text_condition = torch.max(sim_text_condition, dim=1, keepdim=True)[0]
+        sim_text_condition = sim_text_condition / sim_text_condition.max()
+        mask = torch.where(sim_text_condition > 0.3, 0, float('-inf'))
+        mask = mask.repeat(1, image_f.shape[1], 1)
+        image_features = self.model.cross_model(image_f, text_f, mask.half())[:, 0, :]
+
+        image_features = image_features / image_features.norm(dim=-1, keepdim=True)
+        text_features = text_features / text_features.norm(dim=-1, keepdim=True)
+        image_score = self.model.logit_scale.exp() * text_features @ image_features.T
+
+        return image_score[0]

evalscope/metrics/t2v_metrics/models/clipscore_models/pickscore_model.py

@@ -0,0 +1,62 @@
+import torch
+from abc import abstractmethod
+from modelscope import AutoModel, AutoProcessor
+from typing import List
+
+from ...constants import CACHE_DIR
+from ..model import ScoreModel
+
+PICKSCORE_MODELS = ['pickscore-v1']
+
+
+class PickScoreModel(ScoreModel):
+    'A wrapper for PickScore models'
+
+    def __init__(self, model_name='pickscore-v1', device='cuda', cache_dir=CACHE_DIR):
+        assert model_name in PICKSCORE_MODELS
+        super().__init__(model_name=model_name, device=device, cache_dir=cache_dir)
+
+    def load_model(self):
+        """Load the model, tokenizer, image transform
+        """
+        assert self.model_name == 'pickscore-v1'
+        processor_name_or_path = 'laion/CLIP-ViT-H-14-laion2B-s32B-b79K'
+        # model_pretrained_name_or_path = "yuvalkirstain/PickScore_v1"
+        model_pretrained_name_or_path = 'AI-ModelScope/PickScore_v1'  # modelscope model
+
+        self.processor = AutoProcessor.from_pretrained(processor_name_or_path)
+        self.model = AutoModel.from_pretrained(model_pretrained_name_or_path).eval().to(self.device)
+
+    def load_images(self, image: List[str]) -> torch.Tensor:
+        """Load the image(s), and return a tensor (no preprocessing!!) put on self.device
+        """
+        image = [self.image_loader(x) for x in image]
+        image = self.processor(images=image, padding=True, truncation=True, max_length=77,
+                               return_tensors='pt').to(self.device)
+        # image = torch.stack(image, dim=0).to(self.device)
+        return image
+
+    @torch.no_grad()
+    def forward(self, images: List[str], texts: List[str]) -> torch.Tensor:
+        """Forward pass of the model to return n scores for n (image, text) pairs (in PyTorch Tensor)
+        """
+        assert len(images) == len(texts)
+        image = self.load_images(images)
+        text_inputs = self.processor(
+            text=texts,
+            padding=True,
+            truncation=True,
+            max_length=77,
+            return_tensors='pt',
+        ).to(self.device)
+
+        # embed
+        image_embs = self.model.get_image_features(**image)
+        image_embs = image_embs / torch.norm(image_embs, dim=-1, keepdim=True)
+
+        text_embs = self.model.get_text_features(**text_inputs)
+        text_embs = text_embs / torch.norm(text_embs, dim=-1, keepdim=True)
+
+        # score
+        scores = (image_embs * text_embs).sum(dim=-1)
+        return scores

evalscope/metrics/t2v_metrics/models/itmscore_models/__init__.py

@@ -0,0 +1,26 @@
+from ...constants import CACHE_DIR
+from .blip2_itm_model import BLIP2_ITM_MODELS, BLIP2ITMScoreModel
+from .fga_blip2_model import FGA_BLIP2_MODELS, FGA_BLIP2ScoreModel
+from .image_reward_model import IMAGE_REWARD_MODELS, ImageRewardScoreModel
+
+ALL_ITM_MODELS = [
+    BLIP2_ITM_MODELS,
+    IMAGE_REWARD_MODELS,
+    FGA_BLIP2_MODELS,
+]
+
+
+def list_all_itmscore_models():
+    return [model for models in ALL_ITM_MODELS for model in models]
+
+
+def get_itmscore_model(model_name, device='cuda', cache_dir=CACHE_DIR):
+    assert model_name in list_all_itmscore_models()
+    if model_name in BLIP2_ITM_MODELS:
+        return BLIP2ITMScoreModel(model_name, device=device, cache_dir=cache_dir)
+    elif model_name in IMAGE_REWARD_MODELS:
+        return ImageRewardScoreModel(model_name, device=device, cache_dir=cache_dir)
+    elif model_name in FGA_BLIP2_MODELS:
+        return FGA_BLIP2ScoreModel(model_name, device=device, cache_dir=cache_dir)
+    else:
+        raise NotImplementedError()