vislearnlabpy 0.0.2.4__tar.gz → 0.0.2.6__tar.gz

{vislearnlabpy-0.0.2.4/src/vislearnlabpy.egg-info → vislearnlabpy-0.0.2.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: vislearnlabpy
-Version: 0.0.2.4
+Version: 0.0.2.6
 Summary: Visual Learning Lab utility files and pipelines
 Author-email: Tarun Sepuri <tarunsepuri@gmail.com>
 License-Expression: MIT

{vislearnlabpy-0.0.2.4 → vislearnlabpy-0.0.2.6}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "vislearnlabpy"
-version = "0.0.2.4"
+version = "0.0.2.6"
 authors = [
   { name="Tarun Sepuri", email="tarunsepuri@gmail.com" },
 ]

{vislearnlabpy-0.0.2.4 → vislearnlabpy-0.0.2.6}/src/vislearnlabpy/extractions/drawingtask_extractor.py

@@ -76,7 +76,7 @@ class StrokeData:
 class DrawingsExtractor():
     @staticmethod
     def get_default_transformation():
-        return ImageExtractor.get_transformations(apply_center_crop=False, apply_content_crop=True, use_thumbnail=True)
+        return ImageExtractor.get_transformations(apply_center_crop=False, apply_content_crop=True, use_thumbnail=False)
 
     @staticmethod
     def save_transformed(imgData, fname, transform=None):

{vislearnlabpy-0.0.2.4 → vislearnlabpy-0.0.2.6}/src/vislearnlabpy/models/multimodal_model.py

@@ -75,3 +75,42 @@ class MultimodalModel(FeatureGenerator):
         luce = distractor_similarity / (distractor_similarity + target_similarity)
         return luce
 
+    def similarities(self, word1, word2, images):
+        valid_images = [img for img in images if img is not None]
+        similarity_scores = []
+        # TODO: this only returns the similarity scores for the first pair of images: need to separate out, indexing is weird
+        for image1, image2 in itertools.combinations(valid_images, 2):
+            curr_image_embeddings = self.image_embeddings([image1, image2])
+            curr_text_embeddings = self.text_embeddings([word1, word2])
+            # TODO: need to fix how each row is labeled in lookit_similarities, 
+            similarity_scores.append({
+                'image_similarity': self.similarity(curr_image_embeddings[0], curr_image_embeddings[1]),
+                'text_similarity': self.similarity(curr_text_embeddings[0], curr_text_embeddings[1]),
+                # finding distractor image to target word similarity
+                'multimodal_similarity': self.text_to_images_similarity(curr_image_embeddings, curr_text_embeddings[0], logit_scale=10),
+            })
+        if similarity_scores == []:
+            print(f"skipping {word1} and {word2} since they do not have valid images")
+            return [{
+                'image_similarity': None,
+                'text_similarity': None,
+                'multimodal_similarity': None
+            }]
+        else:
+            return similarity_scores
+        
+    # TODO: probably move this to the dataloader row level instead of to a pair of words within a dataloader row
+    # TODO: words or texts? what is my parameter
+    def embeddings(self, word1, word2, dataloader_row):
+        valid_images = [img for img in dataloader_row['images'] if img is not None]
+        output_embeddings = []
+        for image1, image2 in itertools.combinations(valid_images, 2):
+            curr_image_embeddings = self.image_embeddings([image1, image2])
+            curr_text_embeddings = self.text_embeddings([word1, word2])
+            output_embeddings.append({
+                'image_embeddings': curr_image_embeddings,
+                'text_embeddings': curr_text_embeddings,
+                'multimodal_embeddings': self.multimodal_embeddings(curr_image_embeddings, curr_text_embeddings)
+            })
+        return output_embeddings
+    

{vislearnlabpy-0.0.2.4 → vislearnlabpy-0.0.2.6/src/vislearnlabpy.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: vislearnlabpy
-Version: 0.0.2.4
+Version: 0.0.2.6
 Summary: Visual Learning Lab utility files and pipelines
 Author-email: Tarun Sepuri <tarunsepuri@gmail.com>
 License-Expression: MIT