vislearnlabpy 0.0.2.3__tar.gz → 0.0.2.4__tar.gz

{vislearnlabpy-0.0.2.3/src/vislearnlabpy.egg-info → vislearnlabpy-0.0.2.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: vislearnlabpy
-Version: 0.0.2.3
+Version: 0.0.2.4
 Summary: Visual Learning Lab utility files and pipelines
 Author-email: Tarun Sepuri <tarunsepuri@gmail.com>
 License-Expression: MIT

{vislearnlabpy-0.0.2.3 → vislearnlabpy-0.0.2.4}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "vislearnlabpy"
-version = "0.0.2.3"
+version = "0.0.2.4"
 authors = [
   { name="Tarun Sepuri", email="tarunsepuri@gmail.com" },
 ]

{vislearnlabpy-0.0.2.3 → vislearnlabpy-0.0.2.4}/src/vislearnlabpy/embeddings/stimuli_loader.py

@@ -10,6 +10,9 @@ from typing import Callable, Any
 import warnings
 from torchvision import transforms
 import numpy as np
+from scipy import ndimage
+from skimage.morphology import skeletonize, binary_dilation, disk
+from skimage import img_as_bool
 
 random.seed(2)
 
@@ -24,7 +27,12 @@ class ImgExtractionSettings:
     change_stroke_color: bool = False
     stroke_color: tuple = (0, 0, 0)
     stroke_threshold: int = 200
-
+    bg_threshold: int = 200  # threshold to consider a pixel as background (0-255)
+    bg_component_size: int = 10  # minimum size of connected components to keep (in pixels)
+    filter_edge_artifacts: bool = False  # whether to filter out edge artifacts during cropping
+    normalize_stroke_thickness: bool = False
+    stroke_target_thickness: int = 2  # target thickness for stroke normalization
+    double_resize: bool = False  # whether to resize twice (once with thumbnail, once after with resize func)
 
 class ImageExtractor:
     @staticmethod
@@ -87,27 +95,115 @@ class ImageExtractor:
         result[stroke_mask, 2] = target_color[2]  # B
         
         return Image.fromarray(result)
+    
+    def normalize_stroke_thickness(img, target_thickness=2):
+        # Binarize
+        arr = np.array(img.convert('L'))
+        binary = arr < 128  # assuming dark strokes on light background
+        
+        # Skeletonize to 1-pixel lines
+        skeleton = skeletonize(binary)
+        
+        # Dilate to consistent thickness
+        if target_thickness > 1:
+            skeleton = binary_dilation(skeleton, disk(target_thickness//2))
+        
+        # Convert back to image (white background, black strokes)
+        result = (~skeleton * 255).astype(np.uint8)
+        return Image.fromarray(result)
+
+    @staticmethod
+    def _filter_edge_artifacts(large_components, labeled, component_sizes, img_width, img_height):
+        for comp_id in range(1, len(component_sizes)):
+            if large_components[comp_id]:
+                comp_mask = (labeled == comp_id)
+                rows, cols = np.where(comp_mask)
+                
+                if len(rows) > 0 or len(cols) > 0:
+                    ylb, yub = min(rows), max(rows)
+                    xlb, xub = min(cols), max(cols)
+                    width = xub - xlb + 1
+                    height = yub - ylb + 1
+                    x_tolerance = 10
+                    y_tolerance = 100
+                    touches_top = ylb <= y_tolerance
+                    touches_bottom = yub >= img_height - 1 - y_tolerance
+                    touches_left = xlb <= x_tolerance
+                    touches_right = xub >= img_width - 1 - x_tolerance
+                    # Check if it's a thin strip spanning nearly the entire edge
+                    # vertical strip is more likely to not be image artifact
+                    is_thin_vertical_strip = (touches_top or touches_bottom and width < img_width * 0.2)
+                    is_thin_horizontal_strip = (touches_left and touches_right and height < img_height * 0.2)
+                    if is_thin_vertical_strip or is_thin_horizontal_strip or (width / height >= 10) or (height / width >= 10):
+                        large_components[comp_id] = False
+        return large_components
 
     @staticmethod
-    def crop_to_content(img, apply_content_crop):
-        """Crop image to remove white space around content."""
+    def crop_to_content(img, apply_content_crop, stroke_threshold=200, min_component_size=10, filter_edge_artifacts=False):
+        """Crop image to remove white space around content, ignoring small artifacts."""
         if not apply_content_crop:
             return img
             
         arr = np.asarray(img)
-        rows, cols, channels = np.where(arr < 255)
+        img_width, img_height = img.size
+        # Create binary mask of non-white pixels
+        mask = np.all(arr < stroke_threshold, axis=2) if arr.ndim == 3 else arr < stroke_threshold
         
+        # Label connected components
+        labeled, num_features = ndimage.label(mask)
+        
+        # Find component sizes
+        component_sizes = np.bincount(labeled.ravel())
+        # Component 0 is background, so start from 1
+        large_components = component_sizes >= min_component_size
+        large_components[0] = False  # Don't include background
+        # Filter out edge artifacts (components that span entire width or height)
+        if filter_edge_artifacts:
+            large_components = ImageExtractor._filter_edge_artifacts(large_components, labeled, component_sizes, img_width, img_height)
+
+        # Create cleaned mask with only large components
+        cleaned_mask = large_components[labeled]
+        if arr.ndim == 3:
+            white_img = np.ones_like(arr) * 255
+        else:
+            white_img = np.ones_like(arr) * 255
+        
+        # Copy only the content pixels (keep background white)
+        white_img[cleaned_mask] = arr[cleaned_mask]
+        img = Image.fromarray(white_img.astype(np.uint8)) 
         try:
-            xlb = min(cols)  # left bound
-            xub = max(cols)  # right bound  
+            rows, cols = np.where(cleaned_mask)
             ylb = min(rows)  # top bound
             yub = max(rows)  # bottom bound
-            
-            # Make it square by using the same bounds
-            lb = min([xlb, ylb])
-            ub = max([xub, yub])
-            
-            img = img.crop((lb, lb, ub, ub))
+            xlb = min(cols)  # left bound
+            xub = max(cols)  # right bound
+            width = xub - xlb  
+            height = yub - ylb  
+            max_dim = max(width, height) 
+
+            x_center = (xlb + xub) // 2 
+            y_center = (ylb + yub) // 2  
+
+            lb_x = x_center - max_dim // 2
+            ub_x = x_center + max_dim // 2
+            lb_y = y_center - max_dim // 2
+            ub_y = y_center + max_dim // 2
+
+            # Shift if out of bounds (to maintain square)
+            if lb_x < 0:
+                ub_x -= lb_x
+                lb_x = 0
+            if lb_y < 0:
+                ub_y -= lb_y
+                lb_y = 0
+            if ub_x > img.size[0]:
+                lb_x -= (ub_x - img.size[0])
+                ub_x = img.size[0]
+            if ub_y > img.size[1]:
+                lb_y -= (ub_y - img.size[1])
+                ub_y = img.size[1]
+
+            img = img.crop((lb_x, lb_y, ub_x, ub_y))
         except ValueError:
             print('Blank image - skipping crop')
         return img
@@ -124,35 +220,38 @@ class ImageExtractor:
             # Step 1: Apply thumbnail resizing if requested
             if settings.use_thumbnail:
                 image.thumbnail((settings.resize_dim, settings.resize_dim), Image.Resampling.LANCZOS)
+                image.thumbnail((settings.crop_dim, settings.crop_dim), Image.Resampling.LANCZOS)
+            # Step 2: Convert RGBA to RGB
+            img_rgb = ImageExtractor.RGBA2RGB(image)
 
-            # Step 2: Change stroke color if enabled (before converting to RGB)
+            # Step 3: Normalize stroke thickness if enabled
+            if settings.normalize_stroke_thickness:
+                img_rgb = ImageExtractor.normalize_stroke_thickness(
+                    img_rgb, 
+                    settings.stroke_target_thickness
+                )
+            # change stroke color if enabled
             if settings.change_stroke_color:
-                image = ImageExtractor.change_stroke_color(
-                    image, 
+                img_rgb = ImageExtractor.change_stroke_color(
+                    img_rgb, 
                     settings.stroke_color, 
                     settings.stroke_threshold
                 )
-
-            # Step 3: Convert RGBA to RGB
-            img_rgb = ImageExtractor.RGBA2RGB(image)
-            
             # Step 4: Crop to content (remove whitespace) if enabled
-            img_cropped = ImageExtractor.crop_to_content(img_rgb, settings.apply_content_crop)
+            img_cropped = ImageExtractor.crop_to_content(img_rgb, settings.apply_content_crop, settings.bg_threshold, settings.bg_component_size, settings.filter_edge_artifacts)
             return img_cropped
         
         # Build transformation pipeline
         transform_list = []
-        
+        if settings.apply_center_crop:
+            transform_list.append(transforms.CenterCrop(settings.crop_dim))
+        transform_list.append(transforms.Lambda(combined_transform))
+
         # Only add regular resize if not using thumbnail
-        if not settings.use_thumbnail:
-            # resize first
+        if not settings.use_thumbnail or settings.double_resize:
+            # resize last to ensure highest res
             transform_list.append(transforms.Resize(settings.resize_dim))
 
-        transform_list.append(transforms.Lambda(combined_transform))
-        
-        if settings.apply_center_crop:
-            transform_list.append(transforms.CenterCrop(settings.crop_dim))
-        
         # Add any additional transforms you might need
         #transform_list.extend([
             #transforms.ToTensor(),
@@ -210,17 +309,17 @@ class StimuliDataset(Dataset):
             else:
                 with Image.open(image_path) as img:
                     img_copy = img.copy()  # Copy the image data to memory
-                    
-                    if self.transform is not None:
-                        # Apply the transform pipeline
-                        img_copy = self.transform(img_copy)
-                    else:
-                        # Fallback to just converting to RGB
+                    try:
+                        if self.transform is not None:
+                            img_copy = self.transform(img_copy)
+                        else:
+                            img_copy = img_copy.convert('RGB')
+                    except Exception as e:
+                        print(f"Warning: failed to process image {idx} ({e}); skipping transform.")
                         img_copy = img_copy.convert('RGB')
-                    
-                    full_image_paths.append(image_path)
-                    images.append(img_copy)
-        
+                full_image_paths.append(image_path)
+                images.append(img_copy)
+
         # just assuming the image path as an item id for now if there is more than one item per row.
         # TODO: allow for the provision of both item ids and row ids
         row_id = [row[self.id_column] if self.id_column is not None else random.randint(0, 1000000)]

vislearnlabpy-0.0.2.4/src/vislearnlabpy/models/dinov3_model.py

@@ -0,0 +1,20 @@
+from transformers import pipeline
+from vislearnlabpy.models.vision_model import VisionModel
+from transformers import AutoImageProcessor, AutoModel
+import torch
+
+class DinoV3Generator(VisionModel):
+    def __init__(self, dataloader=None, device=None, text_prompt="a photo of a "):
+        self.pipe = pipeline(
+            task="image-feature-extraction",
+            model="facebook/dinov3-vits16-pretrain-lvd1689m",
+            dtype=torch.bfloat16,
+        )
+        self.model, self.preprocess = self.pipe.model, self.pipe.feature_extractor
+        super().__init__(self.model, self.preprocess, dataloader, device)
+        self.name = "dinov3"
+
+    def image_embeddings(self):
+        # 
+        return 
+    

{vislearnlabpy-0.0.2.3 → vislearnlabpy-0.0.2.4}/src/vislearnlabpy/models/multimodal_model.py

@@ -75,42 +75,3 @@ class MultimodalModel(FeatureGenerator):
         luce = distractor_similarity / (distractor_similarity + target_similarity)
         return luce
 
-    def similarities(self, word1, word2, images):
-        valid_images = [img for img in images if img is not None]
-        similarity_scores = []
-        # TODO: this only returns the similarity scores for the first pair of images: need to separate out, indexing is weird
-        for image1, image2 in itertools.combinations(valid_images, 2):
-            curr_image_embeddings = self.image_embeddings([image1, image2])
-            curr_text_embeddings = self.text_embeddings([word1, word2])
-            # TODO: need to fix how each row is labeled in lookit_similarities, 
-            similarity_scores.append({
-                'image_similarity': self.similarity(curr_image_embeddings[0], curr_image_embeddings[1]),
-                'text_similarity': self.similarity(curr_text_embeddings[0], curr_text_embeddings[1]),
-                # finding distractor image to target word similarity
-                'multimodal_similarity': self.text_to_images_similarity(curr_image_embeddings, curr_text_embeddings[0], logit_scale=10),
-            })
-        if similarity_scores == []:
-            print(f"skipping {word1} and {word2} since they do not have valid images")
-            return [{
-                'image_similarity': None,
-                'text_similarity': None,
-                'multimodal_similarity': None
-            }]
-        else:
-            return similarity_scores
-        
-    # TODO: probably move this to the dataloader row level instead of to a pair of words within a dataloader row
-    # TODO: words or texts? what is my parameter
-    def embeddings(self, word1, word2, dataloader_row):
-        valid_images = [img for img in dataloader_row['images'] if img is not None]
-        output_embeddings = []
-        for image1, image2 in itertools.combinations(valid_images, 2):
-            curr_image_embeddings = self.image_embeddings([image1, image2])
-            curr_text_embeddings = self.text_embeddings([word1, word2])
-            output_embeddings.append({
-                'image_embeddings': curr_image_embeddings,
-                'text_embeddings': curr_text_embeddings,
-                'multimodal_embeddings': self.multimodal_embeddings(curr_image_embeddings, curr_text_embeddings)
-            })
-        return output_embeddings
-    

vislearnlabpy-0.0.2.4/src/vislearnlabpy/models/vision_model.py

@@ -0,0 +1,54 @@
+import itertools
+from vislearnlabpy.models.feature_generator import FeatureGenerator
+from vislearnlabpy.embeddings import utils
+from torchvision import transforms
+import torch
+
+class VisionModel(FeatureGenerator):
+    """Abstract base class for multimodal models like CLIP and CVCL that extends FeatureGenerator"""
+    
+    def __init__(self, model, preprocess, dataloader=None, device=None):
+        super().__init__(model, preprocess, dataloader, device)
+        self.image_word_alignment = lambda **x: self.model(**x).logits_per_image.softmax(dim=-1).detach().cpu().numpy()
+
+    # Load and preprocess images
+    def preprocess_image(self, image):
+        if isinstance(image, torch.Tensor):  
+            transform = transforms.ToPILImage()
+            image = transform(image)
+        return self.preprocess(image).unsqueeze(0).to(self.device)
+
+    def encode_image(self, image):
+        return self.model.encode_image(image)
+
+    def image_embeddings(self, images, normalize_embeddings=False):
+        """Get image embeddings (batched for speed)"""
+        # Handle single image case
+        if not isinstance(images, list):
+            return self.image_embeddings([images], normalize_embeddings)[0]
+        # Preprocess all images → batch
+        preprocessed_images = [self.preprocess_image(image) for image in images]
+        preprocessed_images = [image.squeeze(0) if image.dim() == 4 else image for image in preprocessed_images]
+        # Stack into a single tensor batch (assuming tensors are returned)
+        image_batch = torch.stack(preprocessed_images).to(self.device)
+        with torch.no_grad():
+            embeddings = self.encode_image(image_batch)  # model handles batch
+        if normalize_embeddings:
+            embeddings = utils.normalize_embeddings(embeddings)
+        return embeddings
+        
+    # TODO: probably move this to the dataloader row level instead of to a pair of words within a dataloader row
+    # TODO: words or texts? what is my parameter
+    def embeddings(self, word1, word2, dataloader_row):
+        valid_images = [img for img in dataloader_row['images'] if img is not None]
+        output_embeddings = []
+        for image1, image2 in itertools.combinations(valid_images, 2):
+            curr_image_embeddings = self.image_embeddings([image1, image2])
+            curr_text_embeddings = self.text_embeddings([word1, word2])
+            output_embeddings.append({
+                'image_embeddings': curr_image_embeddings,
+                'text_embeddings': curr_text_embeddings,
+                'multimodal_embeddings': self.multimodal_embeddings(curr_image_embeddings, curr_text_embeddings)
+            })
+        return output_embeddings
+    

{vislearnlabpy-0.0.2.3 → vislearnlabpy-0.0.2.4/src/vislearnlabpy.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: vislearnlabpy
-Version: 0.0.2.3
+Version: 0.0.2.4
 Summary: Visual Learning Lab utility files and pipelines
 Author-email: Tarun Sepuri <tarunsepuri@gmail.com>
 License-Expression: MIT

{vislearnlabpy-0.0.2.3 → vislearnlabpy-0.0.2.4}/src/vislearnlabpy.egg-info/SOURCES.txt

@@ -18,5 +18,7 @@ src/vislearnlabpy/embeddings/utils.py
 src/vislearnlabpy/extractions/drawingtask_extractor.py
 src/vislearnlabpy/extractions/mongo_extractor.py
 src/vislearnlabpy/models/clip_model.py
+src/vislearnlabpy/models/dinov3_model.py
 src/vislearnlabpy/models/feature_generator.py
-src/vislearnlabpy/models/multimodal_model.py
+src/vislearnlabpy/models/multimodal_model.py
+src/vislearnlabpy/models/vision_model.py