evalscope 0.17.1__py3-none-any.whl → 1.0.1__py3-none-any.whl

evalscope/metrics/metric.py

@@ -0,0 +1,307 @@
+from collections import defaultdict
+from typing import List
+
+from evalscope.api.metric import Aggregator, AggScore, Metric, SampleScore, T2IMetric
+from evalscope.api.registry import register_aggregation, register_metric
+from .metrics import mean
+
+
+@register_metric(name='exact_match')
+class ExactMatch(Metric):
+
+    def apply(self, predictions, references):
+        return [float(prediction == reference) for prediction, reference in zip(predictions, references)]
+
+
+@register_metric(name='acc')
+class Accuracy(ExactMatch):
+
+    def __init__(self, allow_inclusion: bool = False, numeric: bool = False):
+        self.allow_inclusion = allow_inclusion
+        self.numeric = numeric
+
+    def apply(self, predictions, references):
+        if self.allow_inclusion:
+            results = []
+            for prediction, reference in zip(predictions, references):
+                if prediction and prediction in reference:
+                    results.append(1.0)
+                else:
+                    results.append(0.0)
+            return results
+        elif self.numeric:
+            from .math_parser import extract_answer, math_equal, strip_answer_string
+
+            results = []
+            for prediction, reference in zip(predictions, references):
+                pred_answer = strip_answer_string(extract_answer(prediction))
+                ref_answer = strip_answer_string(reference)
+                results.append(float(math_equal(pred_answer, ref_answer)))
+
+            return results
+        else:
+            return super().apply(predictions, references)
+
+
+@register_metric(name='numeric_match')
+class NumericMatch(Metric):
+
+    def apply(self, predictions, references):
+        return [float(prediction == reference) for prediction, reference in zip(predictions, references)]
+
+
+@register_metric(name='math_acc')
+class MathAcc(Metric):
+
+    def apply(self, predictions, references):
+        from .math_parser import extract_answer, math_equal, strip_answer_string
+
+        results = []
+        for prediction, reference in zip(predictions, references):
+            pred_answer = strip_answer_string(extract_answer(prediction))
+            ref_answer = strip_answer_string(reference)
+            results.append(float(math_equal(pred_answer, ref_answer)))
+
+        return results
+
+
+@register_metric(name='multi_choice_acc')
+class MultiChoiceAcc(Metric):
+
+    def apply(self, predictions, references):
+        """
+        Calculate accuracy for multiple-choice questions.
+
+        Args:
+            predictions (List[str]): List of predicted answers.
+            references (List[str]): List of correct answers.
+
+        Returns:
+            List[float]: List of accuracy scores (1.0 for correct, 0.0 for incorrect).
+        """
+        res = []
+        for prediction, reference in zip(predictions, references):
+            prediction = set(prediction.strip().upper())
+            reference = set(reference.strip().upper())
+            # if the prediction has answer that not in reference, it is wrong
+            if not prediction.issubset(reference):
+                res.append(0.0)
+                continue
+            common = prediction.intersection(reference)
+            res.append(len(common) / len(reference) if reference else 0.0)
+        return res
+
+
+# ##################
+# T2I Metrics ######
+####################
+@register_metric(name='VQAScore')
+class VQAScore(T2IMetric):
+
+    def _init_once(self, model: str = 'clip-flant5-xxl'):
+        from .t2v_metrics.vqascore import VQAScore
+        self.model = VQAScore(model=model)
+
+    def apply(self, images: List[str], texts: List[str], **kwargs) -> List[float]:
+        return self.model(images, texts, **kwargs)
+
+
+@register_metric(name='PickScore')
+class PickScore(T2IMetric):
+
+    def _init_once(self, model: str = 'pickscore-v1'):
+        from .t2v_metrics.clipscore import CLIPScore
+        self.model = CLIPScore(model=model)
+
+    def apply(self, images: List[str], texts: List[str], **kwargs) -> List[float]:
+        return self.model(images, texts, **kwargs)
+
+
+@register_metric(name='CLIPScore')
+class CLIPScore(T2IMetric):
+
+    def _init_once(self, model: str = 'openai:ViT-L-14-336'):
+        from .t2v_metrics.clipscore import CLIPScore
+        self.model = CLIPScore(model=model)
+
+    def apply(self, images: List[str], texts: List[str], **kwargs) -> List[float]:
+        return self.model(images, texts, **kwargs)
+
+
+@register_metric(name='BLIPv2Score')
+class BLIPv2Score(T2IMetric):
+
+    def _init_once(self, model: str = 'blip2-itm'):
+        from .t2v_metrics.itmscore import ITMScore
+        self.model = ITMScore(model=model)
+
+    def apply(self, images: List[str], texts: List[str], **kwargs) -> List[float]:
+        return self.model(images, texts, **kwargs)
+
+
+@register_metric(name='HPSv2Score')
+class HPSv2Score(T2IMetric):
+
+    def _init_once(self, model: str = 'hpsv2'):
+        from .t2v_metrics.clipscore import CLIPScore
+        self.model = CLIPScore(model=model)
+
+    def apply(self, images: List[str], texts: List[str], **kwargs) -> List[float]:
+        return self.model(images, texts, **kwargs)
+
+
+@register_metric(name='HPSv2.1Score')
+class HPSv2_1Score(T2IMetric):
+
+    def _init_once(self, model: str = 'hpsv2.1'):
+        from .t2v_metrics.clipscore import CLIPScore
+        self.model = CLIPScore(model=model)
+
+    def apply(self, images: List[str], texts: List[str], **kwargs) -> List[float]:
+        return self.model(images, texts, **kwargs)
+
+
+@register_metric(name='ImageRewardScore')
+class ImageRewardScore(T2IMetric):
+
+    def _init_once(self, model: str = 'image-reward-v1'):
+        from .t2v_metrics.itmscore import ITMScore
+        self.model = ITMScore(model=model)
+
+    def apply(self, images: List[str], texts: List[str], **kwargs) -> List[float]:
+        return self.model(images, texts, **kwargs)
+
+
+@register_metric(name='FGA_BLIP2Score')
+class FGA_BLIP2Score(T2IMetric):
+
+    def _init_once(self, model: str = 'fga_blip2'):
+        from .t2v_metrics.itmscore import ITMScore
+        self.model = ITMScore(model=model)
+
+    def apply(self, images: List[str], texts: List[str], **kwargs) -> List[float]:
+        return self.model(images, texts, **kwargs)
+
+
+@register_metric(name='MPS')
+class MPS(T2IMetric):
+
+    def _init_once(self, model: str = 'mps'):
+        from .t2v_metrics.clipscore import CLIPScore
+        self.model = CLIPScore(model=model)
+
+    def apply(self, images: List[str], texts: List[str], **kwargs) -> List[float]:
+        return self.model(images, texts, **kwargs)
+
+
+# ##################
+# Aggregators ######
+# ##################
+@register_aggregation(name='mean')
+class Mean(Aggregator):
+
+    name = 'mean'
+
+    def __call__(self, scores: List[SampleScore]) -> List[AggScore]:
+        """Aggregate scores by computing the mean for each metric.
+
+        Args:
+            scores: List of sample scores to aggregate
+
+        Returns:
+            List of aggregated scores with mean values
+        """
+        if not scores:
+            return []
+
+        # Group score values by metric name
+        metric_values = defaultdict(list)
+        metric_sample_ids = defaultdict(list)
+
+        for score in scores:
+
+            for metric_name, value in score.score.value.items():
+                metric_values[metric_name].append(value)
+                metric_sample_ids[metric_name].append(score.sample_id)
+
+        # Calculate mean for each metric
+        aggregated_scores = []
+        for metric_name, values in metric_values.items():
+            if values:  # Only process non-empty value lists
+                aggregated_scores.append(
+                    AggScore(
+                        score=mean(values),
+                        metric_name=metric_name,
+                        aggregation_name=self.name,
+                        num=len(values),
+                        ids=metric_sample_ids[metric_name]
+                    )
+                )
+
+        return aggregated_scores
+
+
+@register_aggregation(name='pass_at_k')
+class PassAtK(Aggregator):
+
+    def __init__(self, k: int = 1):
+        self.k = k
+        self.name = f'pass_at_{k}'
+
+    def __call__(self, scores: List[SampleScore]) -> List[AggScore]:
+        """Aggregate scores by computing the pass@k for each metric using group_id.
+
+        Args:
+            scores: List of sample scores to aggregate
+
+        Returns:
+            List of aggregated scores with pass@k values
+        """
+        if not scores:
+            return []
+
+        import numpy as np
+
+        from .metrics import calculate_pass_at_k
+
+        # Group scores by metric name and group_id
+        metric_groups = defaultdict(lambda: defaultdict(list))
+
+        for score in scores:
+            group_id = getattr(score, 'group_id', score.sample_id)  # fallback to sample_id if no group_id
+
+            for metric_name, value in score.score.value.items():
+                metric_groups[metric_name][group_id].append(float(value))
+
+        # Calculate pass@k for each metric
+        aggregated_scores = []
+        for metric_name, groups in metric_groups.items():
+            if not groups:
+                continue
+
+            # Calculate pass@k for each group (problem)
+            num_samples = []
+            num_correct = []
+            all_sample_ids = []
+
+            for group_id, group_values in groups.items():
+                num_samples.append(len(group_values))
+                num_correct.append(sum(group_values))  # count how many passed in this group
+                all_sample_ids.extend([f'{group_id}_{i}' for i in range(len(group_values))])
+
+            if num_samples:
+                # Use the calculate_pass_at_k function from metrics
+                pass_at_k_values = calculate_pass_at_k(num_samples, num_correct, self.k)
+                overall_pass_at_k = float(np.mean(pass_at_k_values))
+
+                aggregated_scores.append(
+                    AggScore(
+                        score=overall_pass_at_k,
+                        metric_name=f'pass@{self.k}',
+                        aggregation_name='',
+                        num=len(scores),
+                        ids=all_sample_ids
+                    )
+                )
+
+        return aggregated_scores

evalscope/metrics/metrics.py

@@ -191,7 +191,7 @@ def bleu(items):
     return sacrebleu.corpus_bleu(preds, refs).score
 
 
-def bleu_ngram_one_sample(predict, reference):
+def bleu_ngram_one_sample(predict: str, reference: str):
     """
     Calculate BLEU-1, BLEU-2, BLEU-3, and BLEU-4 scores
 
@@ -322,11 +322,11 @@ def bootstrap_stderr(f, xs, iters):
 
     print('bootstrapping for stddev:', f.__name__)
     for bootstrap in tqdm(
-            pool.imap(
-                _bootstrap_internal(f, chunk_size),
-                [(i, xs) for i in range(iters // chunk_size)],
-            ),
-            total=iters // chunk_size,
+        pool.imap(
+            _bootstrap_internal(f, chunk_size),
+            [(i, xs) for i in range(iters // chunk_size)],
+        ),
+        total=iters // chunk_size,
     ):
         # sample w replacement
         res.extend(bootstrap)
@@ -361,15 +361,17 @@ def yesno(x):
         return 'no'
 
 
-def compute_elo(battles,
-                col_model_a='model_a',
-                col_model_b='model_b',
-                col_win='win',
-                tie_values=['tie', 'tie (bothbad)'],
-                k=32,
-                scale=400,
-                base=10,
-                init_rating=1000):
+def compute_elo(
+    battles,
+    col_model_a='model_a',
+    col_model_b='model_b',
+    col_win='win',
+    tie_values=['tie', 'tie (bothbad)'],
+    k=32,
+    scale=400,
+    base=10,
+    init_rating=1000
+):
     rating = defaultdict(lambda: init_rating)
 
     for rd, model_a, model_b, win in battles[[col_model_a, col_model_b, col_win]].itertuples():
@@ -434,9 +436,11 @@ def calculate_arc_accuracy(question_answers: Dict[str, str], predictions: Dict[s
     return score / len(question_answers)
 
 
-def calculate_pass_at_k(num_samples: Union[int, List[int], np.ndarray],
-                        num_correct: Union[List[int], np.ndarray],
-                        k: int = 1) -> np.ndarray:
+def calculate_pass_at_k(
+    num_samples: Union[int, List[int], np.ndarray],
+    num_correct: Union[List[int], np.ndarray],
+    k: int = 1
+) -> np.ndarray:
     """
     Estimates pass@k of each problem and returns them in an array.
     Examples:

evalscope/metrics/t2v_metrics/__init__.py

@@ -1,52 +0,0 @@
-def clip_flant5_score():
-    from .vqascore import VQAScore
-    clip_flant5_score = VQAScore(model='clip-flant5-xxl')
-    return clip_flant5_score
-
-
-def pick_score():
-    from .clipscore import CLIPScore
-    pick_score = CLIPScore(model='pickscore-v1')
-    return pick_score
-
-
-def clip_score():
-    from .clipscore import CLIPScore
-    clip_score = CLIPScore(model='openai:ViT-L-14-336')
-    return clip_score
-
-
-def blip2_score():
-    from .itmscore import ITMScore
-    blip_itm_score = ITMScore(model='blip2-itm')
-    return blip_itm_score
-
-
-def hpsv2_score():
-    from .clipscore import CLIPScore
-    hpsv2_score = CLIPScore(model='hpsv2')
-    return hpsv2_score
-
-
-def hpsv2_1_score():
-    from .clipscore import CLIPScore
-    hpsv2_1_score = CLIPScore(model='hpsv2.1')
-    return hpsv2_1_score
-
-
-def image_reward_score():
-    from .itmscore import ITMScore
-    image_reward_score = ITMScore(model='image-reward-v1')
-    return image_reward_score
-
-
-def fga_blip2_score():
-    from .itmscore import ITMScore
-    fga_blip2_score = ITMScore(model='fga_blip2')
-    return fga_blip2_score
-
-
-def mps_score():
-    from .clipscore import CLIPScore
-    mps_score = CLIPScore(model='mps')
-    return mps_score

evalscope/metrics/t2v_metrics/models/clipscore_models/build_mps_model/clip_model.py

@@ -27,7 +27,8 @@ class XCLIPModel(HFCLIPModel):
         # Use CLIP model's config for some fields (if specified) instead of those of vision & text components.
         output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
         output_hidden_states = (
-            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states)
+            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+        )
         return_dict = return_dict if return_dict is not None else self.config.use_return_dict
 
         text_outputs = self.text_model(
@@ -63,7 +64,8 @@ class XCLIPModel(HFCLIPModel):
         # Use CLIP model's config for some fields (if specified) instead of those of vision & text components.
         output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
         output_hidden_states = (
-            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states)
+            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+        )
         return_dict = return_dict if return_dict is not None else self.config.use_return_dict
 
         vision_outputs = self.vision_model(

evalscope/metrics/t2v_metrics/models/clipscore_models/build_mps_model/cross_modeling.py

@@ -178,15 +178,9 @@ class ParallelTransformerBlock(nn.Module):
 
 class CrossAttention(nn.Module):
 
-    def __init__(self,
-                 dim,
-                 *,
-                 context_dim=None,
-                 dim_head=64,
-                 heads=12,
-                 parallel_ff=False,
-                 ff_mult=4,
-                 norm_context=False):
+    def __init__(
+        self, dim, *, context_dim=None, dim_head=64, heads=12, parallel_ff=False, ff_mult=4, norm_context=False
+    ):
         super().__init__()
         self.heads = heads
         self.scale = dim_head**-0.5
@@ -205,8 +199,8 @@ class CrossAttention(nn.Module):
         ff_inner_dim = ff_mult * dim
 
         self.ff = nn.Sequential(
-            nn.Linear(dim, ff_inner_dim
-                      * 2, bias=False), SwiGLU(), nn.Linear(ff_inner_dim, dim, bias=False)) if parallel_ff else None
+            nn.Linear(dim, ff_inner_dim * 2, bias=False), SwiGLU(), nn.Linear(ff_inner_dim, dim, bias=False)
+        ) if parallel_ff else None
 
     def forward(self, x, context, mask):
         """
@@ -273,9 +267,11 @@ class Cross_model(nn.Module):
             self.layers.append(
                 nn.ModuleList([
                     Residual(
-                        CrossAttention(dim=dim, dim_head=dim_head, heads=heads, parallel_ff=True, ff_mult=ff_mult)),
+                        CrossAttention(dim=dim, dim_head=dim_head, heads=heads, parallel_ff=True, ff_mult=ff_mult)
+                    ),
                     Residual(ParallelTransformerBlock(dim=dim, dim_head=dim_head, heads=heads, ff_mult=ff_mult))
-                ]))
+                ])
+            )
 
     def forward(self, query_tokens, context_tokens, mask):
 

evalscope/metrics/t2v_metrics/models/clipscore_models/clip_model.py

@@ -86,7 +86,8 @@ class CLIPScoreModel(ScoreModel):
         model_file_path = download_open_clip_model(self.arch, self.pretrained, self.cache_dir)
 
         self.model, _, self.preprocess = open_clip.create_model_and_transforms(
-            self.arch, pretrained=model_file_path, device=self.device)
+            self.arch, pretrained=model_file_path, device=self.device
+        )
         self.tokenizer = open_clip.get_tokenizer(self.arch)
         self.model.eval()
 

evalscope/metrics/t2v_metrics/models/clipscore_models/hpsv2_model.py

@@ -44,11 +44,12 @@ class HPSV2ScoreModel(ScoreModel):
             image_std=None,
             image_resize_mode='longest',
             aug_cfg={},
-            output_dict=True)
+            output_dict=True
+        )
 
         # update weight
         model_weight_path = download_file('AI-ModelScope/HPSv2', HPS_VERSION_MAP[self.model_name], self.cache_dir)
-        checkpoint = torch.load(model_weight_path, map_location=self.device)
+        checkpoint = torch.load(model_weight_path, map_location=self.device, weights_only=False)
         self.model.load_state_dict(checkpoint['state_dict'])
         self.tokenizer = open_clip.get_tokenizer(self.arch)
         self.model.eval()

evalscope/metrics/t2v_metrics/models/clipscore_models/mps_model.py

@@ -29,7 +29,8 @@ class MPSModel(ScoreModel):
 
         config = download_file('AI-ModelScope/MPS', file_name='config.json', cache_dir=self.cache_dir)
         model_pretrained_path = download_file(
-            'AI-ModelScope/MPS', file_name='MPS_overall_state_dict.pt', cache_dir=self.cache_dir)  # modelscope model
+            'AI-ModelScope/MPS', file_name='MPS_overall_state_dict.pt', cache_dir=self.cache_dir
+        )  # modelscope model
         model_weight = torch.load(model_pretrained_path, weights_only=True, map_location='cpu')
 
         self.model = CLIPModel(config=CLIPConfig.from_json_file(config))

evalscope/metrics/t2v_metrics/models/clipscore_models/pickscore_model.py

@@ -31,8 +31,8 @@ class PickScoreModel(ScoreModel):
         """Load the image(s), and return a tensor (no preprocessing!!) put on self.device
         """
         image = [self.image_loader(x) for x in image]
-        image = self.processor(
-            images=image, padding=True, truncation=True, max_length=77, return_tensors='pt').to(self.device)
+        image = self.processor(images=image, padding=True, truncation=True, max_length=77,
+                               return_tensors='pt').to(self.device)
         # image = torch.stack(image, dim=0).to(self.device)
         return image
 

evalscope/metrics/t2v_metrics/models/itmscore_models/blip2_itm_model.py

@@ -66,7 +66,8 @@ class BLIP2ITMScoreModel(ScoreModel):
         query_att = torch.ones(query_token.size()[:-1], dtype=torch.long).to(query_token.device)
 
         text_input = self.model.tokenizer(
-            texts, padding='max_length', truncation=True, max_length=35, return_tensors='pt').to(self.device)
+            texts, padding='max_length', truncation=True, max_length=35, return_tensors='pt'
+        ).to(self.device)
 
         attention_mask_all = torch.cat([query_att, text_input.attention_mask], dim=1)
         output_itm = self.model.Qformer.bert(

evalscope/metrics/t2v_metrics/models/itmscore_models/fga_blip2_model.py

@@ -42,10 +42,12 @@ class FGA_BLIP2ScoreModel(ScoreModel):
         # load model
         self.variant = FGA_BLIP2_MODELS[self.model_name]['variant']
         self.model, self.vis_processors, self.text_processors = load_model_and_preprocess(
-            'fga_blip2', self.variant, is_eval=True, device=self.device)
+            'fga_blip2', self.variant, is_eval=True, device=self.device
+        )
         # load pretrained weights
         model_weight_path = download_file(
-            'AI-ModelScope/FGA-BLIP2', file_name='fga_blip2.pth', cache_dir=self.cache_dir)
+            'AI-ModelScope/FGA-BLIP2', file_name='fga_blip2.pth', cache_dir=self.cache_dir
+        )
         self.model.load_checkpoint(model_weight_path)
         self.model.eval()
 

evalscope/metrics/t2v_metrics/models/itmscore_models/image_reward/ImageReward.py

@@ -47,7 +47,8 @@ class MLP(nn.Module):
             nn.Dropout(0.1),
             nn.Linear(64, 16),
             #nn.ReLU(),
-            nn.Linear(16, 1))
+            nn.Linear(16, 1)
+        )
 
         # initial MLP param
         for name, param in self.layers.named_parameters():
@@ -100,7 +101,8 @@ class ImageReward(nn.Module):
 
         # text encode
         text_input = self.blip.tokenizer(
-            prompt, padding='max_length', truncation=True, max_length=35, return_tensors='pt').to(self.device)
+            prompt, padding='max_length', truncation=True, max_length=35, return_tensors='pt'
+        ).to(self.device)
 
         # image encode
         if isinstance(image, Image.Image):
@@ -109,7 +111,8 @@ class ImageReward(nn.Module):
             pil_image = Image.open(image)
         else:
             raise TypeError(
-                r'This image parameter type has not been supportted yet. Please pass PIL.Image or file path str.')
+                r'This image parameter type has not been supportted yet. Please pass PIL.Image or file path str.'
+            )
 
         image = self.preprocess(pil_image).unsqueeze(0).to(self.device)
         image_embeds = self.blip.visual_encoder(image)
@@ -133,7 +136,8 @@ class ImageReward(nn.Module):
     def inference_rank(self, prompt, generations_list):
 
         text_input = self.blip.tokenizer(
-            prompt, padding='max_length', truncation=True, max_length=35, return_tensors='pt').to(self.device)
+            prompt, padding='max_length', truncation=True, max_length=35, return_tensors='pt'
+        ).to(self.device)
 
         txt_set = []
         for generation in generations_list:
@@ -145,7 +149,8 @@ class ImageReward(nn.Module):
                     pil_image = Image.open(generation)
             else:
                 raise TypeError(
-                    r'This image parameter type has not been supportted yet. Please pass PIL.Image or file path str.')
+                    r'This image parameter type has not been supportted yet. Please pass PIL.Image or file path str.'
+                )
             image = self.preprocess(pil_image).unsqueeze(0).to(self.device)
             image_embeds = self.blip.visual_encoder(image)
 

evalscope/metrics/t2v_metrics/models/itmscore_models/image_reward/blip_pretrain.py

@@ -30,7 +30,8 @@ def create_vit(vit, image_size, use_grad_checkpointing=False, ckpt_layer=0, drop
             num_heads=12,
             use_grad_checkpointing=use_grad_checkpointing,
             ckpt_layer=ckpt_layer,
-            drop_path_rate=0 or drop_path_rate)
+            drop_path_rate=0 or drop_path_rate
+        )
     elif vit == 'large':
         vision_width = 1024
         visual_encoder = VisionTransformer(
@@ -41,7 +42,8 @@ def create_vit(vit, image_size, use_grad_checkpointing=False, ckpt_layer=0, drop
             num_heads=16,
             use_grad_checkpointing=use_grad_checkpointing,
             ckpt_layer=ckpt_layer,
-            drop_path_rate=0.1 or drop_path_rate)
+            drop_path_rate=0.1 or drop_path_rate
+        )
     return visual_encoder, vision_width
 
 

evalscope/metrics/t2v_metrics/models/itmscore_models/image_reward_model.py

@@ -53,7 +53,8 @@ class ImageRewardScoreModel(ScoreModel):
         images = self.load_images(images)
         for index in range(len(texts)):
             text_input = self.model.blip.tokenizer(
-                texts[index], padding='max_length', truncation=True, max_length=35, return_tensors='pt').to(self.device)
+                texts[index], padding='max_length', truncation=True, max_length=35, return_tensors='pt'
+            ).to(self.device)
             image_embeds = self.model.blip.visual_encoder(images[index].unsqueeze(0))
             image_atts = torch.ones(image_embeds.size()[:-1], dtype=torch.long).to(self.device)
             text_output = self.model.blip.text_encoder(