evalscope 0.14.0__py3-none-any.whl → 0.15.1__py3-none-any.whl

evalscope/metrics/t2v_metrics/models/clipscore_models/build_mps_model/clip_model.py

@@ -0,0 +1,132 @@
+import torch
+from dataclasses import dataclass
+from torch import einsum, nn
+from transformers import CLIPConfig
+from transformers import CLIPModel as HFCLIPModel
+from typing import Any, Optional, Tuple, Union
+
+from .base_model import BaseModelConfig
+from .cross_modeling import Cross_model
+
+
+class XCLIPModel(HFCLIPModel):
+
+    def __init__(self, config: CLIPConfig):
+        super().__init__(config)
+
+    def get_text_features(
+        self,
+        input_ids: Optional[torch.Tensor] = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.Tensor] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        return_dict: Optional[bool] = None,
+    ) -> torch.FloatTensor:
+
+        # Use CLIP model's config for some fields (if specified) instead of those of vision & text components.
+        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+        output_hidden_states = (
+            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states)
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+        text_outputs = self.text_model(
+            input_ids=input_ids,
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+        )
+
+        # pooled_output = text_outputs[1]
+        # text_features = self.text_projection(pooled_output)
+        last_hidden_state = text_outputs[0]
+        text_features = self.text_projection(last_hidden_state)
+
+        pooled_output = text_outputs[1]
+        text_features_EOS = self.text_projection(pooled_output)
+
+        # del last_hidden_state, text_outputs
+        # gc.collect()
+
+        return text_features, text_features_EOS
+
+    def get_image_features(
+        self,
+        pixel_values: Optional[torch.FloatTensor] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        return_dict: Optional[bool] = None,
+    ) -> torch.FloatTensor:
+
+        # Use CLIP model's config for some fields (if specified) instead of those of vision & text components.
+        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+        output_hidden_states = (
+            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states)
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+        vision_outputs = self.vision_model(
+            pixel_values=pixel_values,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+        )
+
+        # pooled_output = vision_outputs[1]  # pooled_output
+        # image_features = self.visual_projection(pooled_output)
+        last_hidden_state = vision_outputs[0]
+        image_features = self.visual_projection(last_hidden_state)
+
+        return image_features
+
+
+@dataclass
+class ClipModelConfig(BaseModelConfig):
+    _target_: str = 'trainer.models.clip_model.CLIPModel'
+    pretrained_model_name_or_path: str = 'openai/clip-vit-base-patch32'
+
+
+class CLIPModel(nn.Module):
+
+    def __init__(self, config):
+        super().__init__()
+        self.model = XCLIPModel._from_config(config)
+        self.cross_model = Cross_model(dim=1024, layer_num=4, heads=16)
+
+    def get_text_features(self, *args, **kwargs):
+        return self.model.get_text_features(*args, **kwargs)
+
+    def get_image_features(self, *args, **kwargs):
+        return self.model.get_image_features(*args, **kwargs)
+
+    def forward(self, text_inputs=None, image_inputs=None, condition_inputs=None):
+        outputs = ()
+
+        text_f, text_EOS = self.model.get_text_features(text_inputs)  # B*77*1024
+        outputs += text_EOS,
+
+        image_f = self.model.get_image_features(image_inputs.half())  # 2B*257*1024
+        condition_f, _ = self.model.get_text_features(condition_inputs)  # B*5*1024
+
+        sim_text_condition = einsum('b i d, b j d -> b j i', text_f, condition_f)
+        sim_text_condition = torch.max(sim_text_condition, dim=1, keepdim=True)[0]
+        sim_text_condition = sim_text_condition / sim_text_condition.max()
+        mask = torch.where(sim_text_condition > 0.01, 0, float('-inf'))  # B*1*77
+
+        mask = mask.repeat(1, image_f.shape[1], 1)  # B*257*77
+        bc = int(image_f.shape[0] / 2)
+
+        sim0 = self.cross_model(image_f[:bc, :, :], text_f, mask.half())
+        sim1 = self.cross_model(image_f[bc:, :, :], text_f, mask.half())
+        outputs += sim0[:, 0, :],
+        outputs += sim1[:, 0, :],
+
+        return outputs
+
+    @property
+    def logit_scale(self):
+        return self.model.logit_scale
+
+    def save(self, path):
+        self.model.save_pretrained(path)

evalscope/metrics/t2v_metrics/models/clipscore_models/build_mps_model/cross_modeling.py

@@ -0,0 +1,286 @@
+import torch
+import torch.nn.functional as F
+from einops import rearrange, repeat
+from torch import einsum, nn
+
+# helper functions
+
+
+def exists(val):
+    return val is not None
+
+
+def default(val, d):
+    return val if exists(val) else d
+
+
+# normalization
+# they use layernorm without bias, something that pytorch does not offer
+
+
+class LayerNorm(nn.Module):
+
+    def __init__(self, dim):
+        super().__init__()
+        self.weight = nn.Parameter(torch.ones(dim))
+        self.register_buffer('bias', torch.zeros(dim))
+
+    def forward(self, x):
+        return F.layer_norm(x, x.shape[-1:], self.weight, self.bias)
+
+
+# residual
+
+
+class Residual(nn.Module):
+
+    def __init__(self, fn):
+        super().__init__()
+        self.fn = fn
+
+    def forward(self, x, *args, **kwargs):
+        return self.fn(x, *args, **kwargs) + x
+
+
+# rotary positional embedding
+# https://arxiv.org/abs/2104.09864
+
+
+class RotaryEmbedding(nn.Module):
+
+    def __init__(self, dim):
+        super().__init__()
+        inv_freq = 1.0 / (10000**(torch.arange(0, dim, 2).float() / dim))
+        self.register_buffer('inv_freq', inv_freq)
+
+    def forward(self, max_seq_len, *, device):
+        seq = torch.arange(max_seq_len, device=device, dtype=self.inv_freq.dtype)
+        freqs = einsum('i , j -> i j', seq, self.inv_freq)
+        return torch.cat((freqs, freqs), dim=-1)
+
+
+def rotate_half(x):
+    x = rearrange(x, '... (j d) -> ... j d', j=2)
+    x1, x2 = x.unbind(dim=-2)
+    return torch.cat((-x2, x1), dim=-1)
+
+
+def apply_rotary_pos_emb(pos, t):
+    return (t * pos.cos()) + (rotate_half(t) * pos.sin())
+
+
+# classic Noam Shazeer paper, except here they use SwiGLU instead of the more popular GEGLU for gating the feedforward
+# https://arxiv.org/abs/2002.05202
+
+
+class SwiGLU(nn.Module):
+
+    def forward(self, x):
+        x, gate = x.chunk(2, dim=-1)
+        return F.silu(gate) * x
+
+
+# parallel attention and feedforward with residual
+# discovered by Wang et al + EleutherAI from GPT-J fame
+
+
+class ParallelTransformerBlock(nn.Module):
+
+    def __init__(self, dim, dim_head=64, heads=8, ff_mult=4):
+        super().__init__()
+        self.norm = LayerNorm(dim)
+
+        attn_inner_dim = dim_head * heads
+        ff_inner_dim = dim * ff_mult
+        self.fused_dims = (attn_inner_dim, dim_head, dim_head, (ff_inner_dim * 2))
+
+        self.heads = heads
+        self.scale = dim_head**-0.5
+        self.rotary_emb = RotaryEmbedding(dim_head)
+
+        self.fused_attn_ff_proj = nn.Linear(dim, sum(self.fused_dims), bias=False)
+        self.attn_out = nn.Linear(attn_inner_dim, dim, bias=False)
+
+        self.ff_out = nn.Sequential(SwiGLU(), nn.Linear(ff_inner_dim, dim, bias=False))
+
+        self.register_buffer('pos_emb', None, persistent=False)
+
+    def get_rotary_embedding(self, n, device):
+        if self.pos_emb is not None and self.pos_emb.shape[-2] >= n:
+            return self.pos_emb[:n]
+
+        pos_emb = self.rotary_emb(n, device=device)
+        self.register_buffer('pos_emb', pos_emb, persistent=False)
+        return pos_emb
+
+    def forward(self, x, attn_mask=None):
+        """
+        einstein notation
+        b - batch
+        h - heads
+        n, i, j - sequence length (base sequence length, source, target)
+        d - feature dimension
+        """
+
+        n, device, h = x.shape[1], x.device, self.heads
+
+        # pre layernorm
+
+        x = self.norm(x)
+
+        # attention queries, keys, values, and feedforward inner
+
+        q, k, v, ff = self.fused_attn_ff_proj(x).split(self.fused_dims, dim=-1)
+
+        # split heads
+        # they use multi-query single-key-value attention, yet another Noam Shazeer paper
+        # they found no performance loss past a certain scale, and more efficient decoding obviously
+        # https://arxiv.org/abs/1911.02150
+
+        q = rearrange(q, 'b n (h d) -> b h n d', h=h)
+
+        # rotary embeddings
+
+        positions = self.get_rotary_embedding(n, device)
+        q, k = map(lambda t: apply_rotary_pos_emb(positions, t), (q, k))
+
+        # scale
+
+        q = q * self.scale
+
+        # similarity
+
+        sim = einsum('b h i d, b j d -> b h i j', q, k)
+
+        # extra attention mask - for masking out attention from text CLS token to padding
+
+        if exists(attn_mask):
+            attn_mask = rearrange(attn_mask, 'b i j -> b 1 i j')
+            sim = sim.masked_fill(~attn_mask, -torch.finfo(sim.dtype).max)
+
+        # attention
+
+        sim = sim - sim.amax(dim=-1, keepdim=True).detach()
+        attn = sim.softmax(dim=-1)
+
+        # aggregate values
+
+        out = einsum('b h i j, b j d -> b h i d', attn, v)
+
+        # merge heads
+
+        out = rearrange(out, 'b h n d -> b n (h d)')
+        return self.attn_out(out) + self.ff_out(ff)
+
+
+# cross attention - using multi-query + one-headed key / values as in PaLM w/ optional parallel feedforward
+
+
+class CrossAttention(nn.Module):
+
+    def __init__(self,
+                 dim,
+                 *,
+                 context_dim=None,
+                 dim_head=64,
+                 heads=12,
+                 parallel_ff=False,
+                 ff_mult=4,
+                 norm_context=False):
+        super().__init__()
+        self.heads = heads
+        self.scale = dim_head**-0.5
+        inner_dim = heads * dim_head
+        context_dim = default(context_dim, dim)
+
+        self.norm = LayerNorm(dim)
+        self.context_norm = LayerNorm(context_dim) if norm_context else nn.Identity()
+
+        self.to_q = nn.Linear(dim, inner_dim, bias=False)
+        self.to_kv = nn.Linear(context_dim, dim_head * 2, bias=False)
+        self.to_out = nn.Linear(inner_dim, dim, bias=False)
+
+        # whether to have parallel feedforward
+
+        ff_inner_dim = ff_mult * dim
+
+        self.ff = nn.Sequential(
+            nn.Linear(dim, ff_inner_dim
+                      * 2, bias=False), SwiGLU(), nn.Linear(ff_inner_dim, dim, bias=False)) if parallel_ff else None
+
+    def forward(self, x, context, mask):
+        """
+        einstein notation
+        b - batch
+        h - heads
+        n, i, j - sequence length (base sequence length, source, target)
+        d - feature dimension
+        """
+
+        # pre-layernorm, for queries and context
+
+        x = self.norm(x)
+        context = self.context_norm(context)
+
+        # get queries
+
+        q = self.to_q(x)
+        q = rearrange(q, 'b n (h d) -> b h n d', h=self.heads)
+
+        # scale
+
+        q = q * self.scale
+
+        # get key / values
+
+        k, v = self.to_kv(context).chunk(2, dim=-1)
+
+        # query / key similarity
+
+        sim = einsum('b h i d, b j d -> b h i j', q, k)
+
+        # attention
+        mask = mask.unsqueeze(1).repeat(1, self.heads, 1, 1)
+        sim = sim + mask  # context mask
+        sim = sim - sim.amax(dim=-1, keepdim=True)
+        attn = sim.softmax(dim=-1)
+
+        # aggregate
+
+        out = einsum('b h i j, b j d -> b h i d', attn, v)
+
+        # merge and combine heads
+
+        out = rearrange(out, 'b h n d -> b n (h d)')
+        out = self.to_out(out)
+
+        # add parallel feedforward (for multimodal layers)
+
+        if exists(self.ff):
+            out = out + self.ff(x)
+
+        return out
+
+
+class Cross_model(nn.Module):
+
+    def __init__(self, dim=512, layer_num=4, dim_head=64, heads=8, ff_mult=4):
+        super().__init__()
+
+        self.layers = nn.ModuleList([])
+
+        for ind in range(layer_num):
+            self.layers.append(
+                nn.ModuleList([
+                    Residual(
+                        CrossAttention(dim=dim, dim_head=dim_head, heads=heads, parallel_ff=True, ff_mult=ff_mult)),
+                    Residual(ParallelTransformerBlock(dim=dim, dim_head=dim_head, heads=heads, ff_mult=ff_mult))
+                ]))
+
+    def forward(self, query_tokens, context_tokens, mask):
+
+        for cross_attn, self_attn_ff in self.layers:
+            query_tokens = cross_attn(query_tokens, context_tokens, mask)
+            query_tokens = self_attn_ff(query_tokens)
+
+        return query_tokens

evalscope/metrics/t2v_metrics/models/clipscore_models/clip_model.py

@@ -0,0 +1,114 @@
+import os
+import torch
+from typing import List
+
+from ...constants import CACHE_DIR
+from ..model import ScoreModel
+
+CLIP_MODELS = [
+    'openai:RN50', 'yfcc15m:RN50', 'cc12m:RN50', 'openai:RN101', 'yfcc15m:RN101', 'openai:RN50x4', 'openai:RN50x16',
+    'openai:RN50x64', 'openai:ViT-B-32', 'laion400m_e31:ViT-B-32', 'laion400m_e32:ViT-B-32', 'laion2b_e16:ViT-B-32',
+    'laion2b_s34b_b79k:ViT-B-32', 'datacomp_xl_s13b_b90k:ViT-B-32', 'datacomp_m_s128m_b4k:ViT-B-32',
+    'commonpool_m_clip_s128m_b4k:ViT-B-32', 'commonpool_m_laion_s128m_b4k:ViT-B-32',
+    'commonpool_m_image_s128m_b4k:ViT-B-32', 'commonpool_m_text_s128m_b4k:ViT-B-32',
+    'commonpool_m_basic_s128m_b4k:ViT-B-32', 'commonpool_m_s128m_b4k:ViT-B-32', 'datacomp_s_s13m_b4k:ViT-B-32',
+    'commonpool_s_clip_s13m_b4k:ViT-B-32', 'commonpool_s_laion_s13m_b4k:ViT-B-32',
+    'commonpool_s_image_s13m_b4k:ViT-B-32', 'commonpool_s_text_s13m_b4k:ViT-B-32',
+    'commonpool_s_basic_s13m_b4k:ViT-B-32', 'commonpool_s_s13m_b4k:ViT-B-32', 'metaclip_400m:ViT-B-32',
+    'metaclip_fullcc:ViT-B-32', 'datacomp_s34b_b86k:ViT-B-32-256', 'openai:ViT-B-16', 'laion400m_e31:ViT-B-16',
+    'laion400m_e32:ViT-B-16', 'laion2b_s34b_b88k:ViT-B-16', 'datacomp_xl_s13b_b90k:ViT-B-16',
+    'datacomp_l_s1b_b8k:ViT-B-16', 'commonpool_l_clip_s1b_b8k:ViT-B-16', 'commonpool_l_laion_s1b_b8k:ViT-B-16',
+    'commonpool_l_image_s1b_b8k:ViT-B-16', 'commonpool_l_text_s1b_b8k:ViT-B-16', 'commonpool_l_basic_s1b_b8k:ViT-B-16',
+    'commonpool_l_s1b_b8k:ViT-B-16', 'dfn2b:ViT-B-16', 'metaclip_400m:ViT-B-16', 'metaclip_fullcc:ViT-B-16',
+    'laion400m_e31:ViT-B-16-plus-240', 'laion400m_e32:ViT-B-16-plus-240', 'openai:ViT-L-14', 'laion400m_e31:ViT-L-14',
+    'laion400m_e32:ViT-L-14', 'laion2b_s32b_b82k:ViT-L-14', 'datacomp_xl_s13b_b90k:ViT-L-14',
+    'commonpool_xl_clip_s13b_b90k:ViT-L-14', 'commonpool_xl_laion_s13b_b90k:ViT-L-14',
+    'commonpool_xl_s13b_b90k:ViT-L-14', 'metaclip_400m:ViT-L-14', 'metaclip_fullcc:ViT-L-14', 'dfn2b:ViT-L-14',
+    'dfn2b_s39b:ViT-L-14', 'openai:ViT-L-14-336', 'laion2b_s32b_b79k:ViT-H-14', 'metaclip_fullcc:ViT-H-14',
+    'metaclip_altogether:ViT-H-14', 'dfn5b:ViT-H-14', 'dfn5b:ViT-H-14-378', 'laion2b_s12b_b42k:ViT-g-14',
+    'laion2b_s34b_b88k:ViT-g-14', 'laion2b_s39b_b160k:ViT-bigG-14', 'metaclip_fullcc:ViT-bigG-14',
+    'laion2b_s12b_b32k:roberta-ViT-B-32', 'laion5b_s13b_b90k:xlm-roberta-base-ViT-B-32',
+    'frozen_laion5b_s13b_b90k:xlm-roberta-large-ViT-H-14', 'laion400m_s13b_b51k:convnext_base',
+    'laion2b_s13b_b82k:convnext_base_w', 'laion2b_s13b_b82k_augreg:convnext_base_w',
+    'laion_aesthetic_s13b_b82k:convnext_base_w', 'laion_aesthetic_s13b_b82k:convnext_base_w_320',
+    'laion_aesthetic_s13b_b82k_augreg:convnext_base_w_320', 'laion2b_s26b_b102k_augreg:convnext_large_d',
+    'laion2b_s29b_b131k_ft:convnext_large_d_320', 'laion2b_s29b_b131k_ft_soup:convnext_large_d_320',
+    'laion2b_s34b_b82k_augreg:convnext_xxlarge', 'laion2b_s34b_b82k_augreg_rewind:convnext_xxlarge',
+    'laion2b_s34b_b82k_augreg_soup:convnext_xxlarge', 'laion2b_s13b_b90k:coca_ViT-B-32',
+    'mscoco_finetuned_laion2b_s13b_b90k:coca_ViT-B-32', 'laion2b_s13b_b90k:coca_ViT-L-14',
+    'mscoco_finetuned_laion2b_s13b_b90k:coca_ViT-L-14', 'laion400m_s11b_b41k:EVA01-g-14',
+    'merged2b_s11b_b114k:EVA01-g-14-plus', 'merged2b_s8b_b131k:EVA02-B-16', 'merged2b_s4b_b131k:EVA02-L-14',
+    'merged2b_s6b_b61k:EVA02-L-14-336', 'laion2b_s4b_b115k:EVA02-E-14', 'laion2b_s9b_b144k:EVA02-E-14-plus',
+    'webli:ViT-B-16-SigLIP', 'webli:ViT-B-16-SigLIP-256', 'webli:ViT-B-16-SigLIP-i18n-256', 'webli:ViT-B-16-SigLIP-384',
+    'webli:ViT-B-16-SigLIP-512', 'webli:ViT-L-16-SigLIP-256', 'webli:ViT-L-16-SigLIP-384', 'webli:ViT-SO400M-14-SigLIP',
+    'webli:ViT-SO400M-16-SigLIP-i18n-256', 'webli:ViT-SO400M-14-SigLIP-378', 'webli:ViT-SO400M-14-SigLIP-384',
+    'webli:ViT-B-32-SigLIP2-256', 'webli:ViT-B-16-SigLIP2', 'webli:ViT-B-16-SigLIP2-256', 'webli:ViT-B-16-SigLIP2-384',
+    'webli:ViT-B-16-SigLIP2-512', 'webli:ViT-L-16-SigLIP2-256', 'webli:ViT-L-16-SigLIP2-384',
+    'webli:ViT-L-16-SigLIP2-512', 'webli:ViT-SO400M-14-SigLIP2', 'webli:ViT-SO400M-14-SigLIP2-378',
+    'webli:ViT-SO400M-16-SigLIP2-256', 'webli:ViT-SO400M-16-SigLIP2-384', 'webli:ViT-SO400M-16-SigLIP2-512',
+    'webli:ViT-gopt-16-SigLIP2-256', 'webli:ViT-gopt-16-SigLIP2-384', 'datacomp1b:ViT-L-14-CLIPA',
+    'datacomp1b:ViT-L-14-CLIPA-336', 'datacomp1b:ViT-H-14-CLIPA', 'laion2b:ViT-H-14-CLIPA-336',
+    'datacomp1b:ViT-H-14-CLIPA-336', 'datacomp1b:ViT-bigG-14-CLIPA', 'datacomp1b:ViT-bigG-14-CLIPA-336',
+    'v1:nllb-clip-base', 'v1:nllb-clip-large', 'v1:nllb-clip-base-siglip', 'mrl:nllb-clip-base-siglip',
+    'v1:nllb-clip-large-siglip', 'mrl:nllb-clip-large-siglip', 'datacompdr:MobileCLIP-S1', 'datacompdr:MobileCLIP-S2',
+    'datacompdr:MobileCLIP-B', 'datacompdr_lt:MobileCLIP-B', 'datacomp1b:ViTamin-S', 'datacomp1b:ViTamin-S-LTT',
+    'datacomp1b:ViTamin-B', 'datacomp1b:ViTamin-B-LTT', 'datacomp1b:ViTamin-L', 'datacomp1b:ViTamin-L-256',
+    'datacomp1b:ViTamin-L-336', 'datacomp1b:ViTamin-L-384', 'datacomp1b:ViTamin-L2', 'datacomp1b:ViTamin-L2-256',
+    'datacomp1b:ViTamin-L2-336', 'datacomp1b:ViTamin-L2-384', 'datacomp1b:ViTamin-XL-256', 'datacomp1b:ViTamin-XL-336',
+    'datacomp1b:ViTamin-XL-384', 'openai:RN50-quickgelu', 'yfcc15m:RN50-quickgelu', 'cc12m:RN50-quickgelu',
+    'openai:RN101-quickgelu', 'yfcc15m:RN101-quickgelu', 'openai:RN50x4-quickgelu', 'openai:RN50x16-quickgelu',
+    'openai:RN50x64-quickgelu', 'openai:ViT-B-32-quickgelu', 'laion400m_e31:ViT-B-32-quickgelu',
+    'laion400m_e32:ViT-B-32-quickgelu', 'metaclip_400m:ViT-B-32-quickgelu', 'metaclip_fullcc:ViT-B-32-quickgelu',
+    'openai:ViT-B-16-quickgelu', 'dfn2b:ViT-B-16-quickgelu', 'metaclip_400m:ViT-B-16-quickgelu',
+    'metaclip_fullcc:ViT-B-16-quickgelu', 'openai:ViT-L-14-quickgelu', 'metaclip_400m:ViT-L-14-quickgelu',
+    'metaclip_fullcc:ViT-L-14-quickgelu', 'dfn2b:ViT-L-14-quickgelu', 'openai:ViT-L-14-336-quickgelu',
+    'metaclip_fullcc:ViT-H-14-quickgelu', 'dfn5b:ViT-H-14-quickgelu', 'dfn5b:ViT-H-14-378-quickgelu',
+    'metaclip_fullcc:ViT-bigG-14-quickgelu'
+]  # noqa: E501
+
+
+class CLIPScoreModel(ScoreModel):
+    "A wrapper for OpenCLIP models (including openAI's CLIP, OpenCLIP, DatacompCLIP)"
+
+    def __init__(self, model_name='openai:ViT-L-14', device='cuda', cache_dir=CACHE_DIR):
+        assert model_name in CLIP_MODELS
+        super().__init__(model_name=model_name, device=device, cache_dir=cache_dir)
+
+    def load_model(self):
+        """Load the model, tokenizer, image transform
+        """
+        import open_clip
+
+        from ..utils import download_open_clip_model
+
+        self.pretrained, self.arch = self.model_name.split(':')
+        # load model from modelscope
+        model_file_path = download_open_clip_model(self.arch, self.pretrained, self.cache_dir)
+
+        self.model, _, self.preprocess = open_clip.create_model_and_transforms(
+            self.arch, pretrained=model_file_path, device=self.device)
+        self.tokenizer = open_clip.get_tokenizer(self.arch)
+        self.model.eval()
+
+    def load_images(self, image: List[str]) -> torch.Tensor:
+        """Load the image(s), and return a tensor (after preprocessing) put on self.device
+        """
+        image = [self.image_loader(x) for x in image]
+        image = [self.preprocess(x) for x in image]
+        image = torch.stack(image, dim=0).to(self.device)
+        return image
+
+    @torch.no_grad()
+    def forward(self, images: List[str], texts: List[str]) -> torch.Tensor:
+        """Forward pass of the model to return n scores for n (image, text) pairs (in PyTorch Tensor)
+        """
+        assert len(images) == len(texts)
+        image = self.load_images(images)
+        text = self.tokenizer(texts).to(self.device)
+        image_features = self.model.encode_image(image)
+        image_features /= image_features.norm(dim=-1, keepdim=True)
+        text_features = self.model.encode_text(text)
+        text_features /= text_features.norm(dim=-1, keepdim=True)
+
+        # return cosine similarity as scores
+        return (image_features * text_features).sum(dim=-1)

evalscope/metrics/t2v_metrics/models/clipscore_models/hpsv2_model.py

@@ -0,0 +1,86 @@
+import torch
+from typing import List
+
+from ...constants import CACHE_DIR
+from ..model import ScoreModel
+
+HPSV2_MODELS = ['hpsv2', 'hpsv2.1']
+HPS_VERSION_MAP = {
+    'hpsv2': 'HPS_v2_compressed.pt',
+    'hpsv2.1': 'HPS_v2.1_compressed.pt',
+}
+
+
+class HPSV2ScoreModel(ScoreModel):
+    'A wrapper for HPSv2 models '
+
+    def __init__(self, model_name='openai:ViT-L-14', device='cuda', cache_dir=CACHE_DIR):
+        assert model_name in HPSV2_MODELS
+        super().__init__(model_name=model_name, device=device, cache_dir=cache_dir)
+
+    def load_model(self):
+        """Load the model, tokenizer, image transform
+        """
+        import open_clip
+
+        from ..utils import download_file, download_open_clip_model
+
+        self.pretrained, self.arch = 'laion2B-s32B-b79K:ViT-H-14'.split(':')
+        # load model from modelscope
+        model_file_path = download_open_clip_model(self.arch, self.pretrained, self.cache_dir)
+
+        self.model, _, self.preprocess = open_clip.create_model_and_transforms(
+            self.arch,
+            pretrained=model_file_path,
+            precision='amp',
+            device=self.device,
+            jit=False,
+            force_quick_gelu=False,
+            force_custom_text=False,
+            force_patch_dropout=False,
+            force_image_size=None,
+            pretrained_image=False,
+            image_mean=None,
+            image_std=None,
+            image_resize_mode='longest',
+            aug_cfg={},
+            output_dict=True)
+
+        # update weight
+        model_weight_path = download_file('AI-ModelScope/HPSv2', HPS_VERSION_MAP[self.model_name], self.cache_dir)
+        checkpoint = torch.load(model_weight_path, map_location=self.device)
+        self.model.load_state_dict(checkpoint['state_dict'])
+        self.tokenizer = open_clip.get_tokenizer(self.arch)
+        self.model.eval()
+
+    def load_images(self, image: List[str]):
+        """Load the image(s), and return a tensor (after preprocessing) put on self.device
+        """
+        images = [self.image_loader(x) for x in image]
+        return images
+
+    @torch.no_grad()
+    def forward(self, images: List[str], texts: List[str]) -> torch.Tensor:
+        """Forward pass of the model to return n scores for n (image, text) pairs (in PyTorch Tensor)
+        """
+        assert len(images) == len(texts)
+        images = self.load_images(images)
+        scores = torch.zeros(len(images), dtype=torch.float16).to(self.device)
+        for i in range(len(images)):
+            caption = texts[i]
+            image = images[i]
+            # Process the image
+            image = self.preprocess(image).unsqueeze(0).to(device=self.device, non_blocking=True)
+            # Process the prompt
+            text = self.tokenizer([caption]).to(device=self.device, non_blocking=True)  # Updated to use texts[i]
+            # Calculate the HPS
+            with torch.amp.autocast(device_type=self.device):
+                outputs = self.model(image, text)
+                image_features, text_features = outputs['image_features'], outputs['text_features']
+                logits_per_image = image_features @ text_features.T
+
+                hps_score = torch.diagonal(logits_per_image).cpu().numpy()
+            scores[i] = float(hps_score[0])
+
+        # return cosine similarity as scores
+        return scores