brainscore-vision 2.2.3__py3-none-any.whl → 2.2.5__py3-none-any.whl

brainscore_vision/models/resnet50_VITO_8deg_cc/helpers/spatialattn.py

@@ -0,0 +1,50 @@
+import torch
+from torch import nn
+
+class DSConv2d(nn.Module):
+	def __init__(self, nin, kernels_per_layer, nout): 
+		super(DSConv2d, self).__init__() 
+		self.depthwise = nn.Conv2d(nin, nin * kernels_per_layer, kernel_size=3, padding=1, groups=nin, padding_mode='reflect') 
+		self.pointwise = nn.Conv2d(nin * kernels_per_layer, nout, kernel_size=1) 
+  
+	def forward(self, x): 
+		out = self.depthwise(x) 
+		out = self.pointwise(out) 
+		return out
+
+
+class SpatialAttn(nn.Module):
+	def __init__(self, inp_dim, hidden_sizes, num_heads, atttype='channel'):
+		super(SpatialAttn, self).__init__() 
+		self.inp_dim = inp_dim
+		self.hidden_sizes = hidden_sizes
+		self.num_heads = num_heads
+		
+		self.module_list = nn.ModuleList([])
+		if atttype == 'channel':
+			first_pool = nn.Conv2d(self.inp_dim, hidden_sizes[0], kernel_size=1, bias=True)
+		elif atttype == 'dw':
+			first_pool = DSConv2d(self.inp_dim, 1, hidden_sizes[0])
+		for i in range(len(hidden_sizes)+1):
+			if i == 0:
+				self.module_list.append(first_pool)
+				self.module_list.append(nn.ReLU(inplace=True))
+			elif i < len(hidden_sizes):
+				self.module_list.append(nn.Conv2d(hidden_sizes[i-1], hidden_sizes[i], kernel_size=1, bias=True))
+				self.module_list.append(nn.ReLU(inplace=True))
+			else:
+				self.module_list.append(nn.Conv2d(hidden_sizes[i-1], num_heads, kernel_size=1, bias=False))
+		
+
+	def forward(self, x):
+		inp=x
+		for m in self.module_list:
+			x = m(x)
+		assert self.inp_dim % x.shape[1] == 0, "Output channels must be divisible by input number of channels"
+		x = nn.Softmax(dim=-1)(x.view(x.shape[0], x.shape[1], -1)).view(x.shape)
+		if x.shape[1] != self.inp_dim:
+			x = x.repeat(1, self.inp_dim // x.shape[1], 1, 1)
+		w = x
+		x = w*inp
+		x = x.sum((-1,-2))
+		return x

brainscore_vision/models/resnet50_VITO_8deg_cc/model.py

@@ -0,0 +1,72 @@
+from brainscore_vision.model_helpers.check_submission import check_models
+from brainscore_vision.model_helpers.activations.pytorch import PytorchWrapper
+from brainscore_vision.model_helpers.activations.pytorch import load_preprocess_images
+import functools
+import torch
+import os
+from brainscore_vision.model_helpers.s3 import load_weight_file
+from brainscore_vision.model_helpers.brain_transformation import ModelCommitment
+from .helpers.resnet import resnet50
+
+dir_path = os.path.dirname(os.path.realpath(__file__))
+
+"""
+Template module for a base model submission to brain-score
+"""
+
+LAYERS = ['layer1.0', 'layer1.1', 'layer1.2', 'layer2.0', 'layer2.1', 'layer2.2', 'layer2.3', 'layer3.0',
+                   'layer3.1', 'layer3.2', 'layer3.3', 'layer3.4', 'layer3.5', 'layer4.0', 'layer4.1', 'layer4.2',
+                   'avgpool', 'fc']
+
+def get_model(name):
+    """
+    This method fetches an instance of a base model. The instance has to be callable and return a xarray object,
+    containing activations. There exist standard wrapper implementations for common libraries, like pytorch and
+    keras. Checkout the examples folder, to see more. For custom implementations check out the implementation of the
+    wrappers.
+    :param name: the name of the model to fetch
+    :return: the model instance
+    """
+
+    assert name == 'resnet50-VITO-8deg-cc'
+    model = resnet50(attn_pool=True)
+
+    weights_path = load_weight_file(bucket="brainscore-storage", folder_name="brainscore-vision/models",
+                                    relative_path="resnet50-VITO-8deg-cc/resnet50_vito_linear_sdattn.pth",
+                                    version_id="null",
+                                    sha1="2103a1941017fdb5f58ac77e5587e7a73cd96bd8")
+    r50_vito_sd = torch.load(weights_path, map_location='cpu')
+    model.load_state_dict(r50_vito_sd, strict=False)
+    model.eval()
+    preprocessing = functools.partial(load_preprocess_images, image_size=224)
+    wrapper = PytorchWrapper(identifier='resnet50-VITO-8deg-cc', model=model, preprocessing=preprocessing)
+    #layers = LAYERS
+    #vito_brain_model = ModelCommitment('resnet50-VITO-8deg-cc', activations_model=wrapper, layers=layers,
+    #                                   visual_degrees=8)
+    return wrapper
+
+
+def get_layers(name):
+    assert name == 'resnet50-VITO-8deg-cc'
+    """
+    This method returns a list of string layer names to consider per model. The benchmarks maps brain regions to
+    layers and uses this list as a set of possible layers. The lists doesn't have to contain all layers, the less the
+    faster the benchmark process works. Additionally the given layers have to produce an activations vector of at least
+    size 25! The layer names are delivered back to the model instance and have to be resolved in there. For a pytorch
+    model, the layer name are for instance dot concatenated per module, e.g. "features.2".
+    :param name: the name of the model, to return the layers for
+    :return: a list of strings containing all layers, that should be considered as brain area.
+    """
+    return LAYERS
+
+
+def get_bibtex(model_identifier):
+
+    """
+    A method returning the bibtex reference of the requested model as a string.
+    """
+    return ''
+
+
+if __name__ == '__main__':
+    check_models.check_base_models(__name__)

brainscore_vision/models/resnet50_VITO_8deg_cc/region_layer_map/resnet50-VITO-8deg-cc.json

@@ -0,0 +1,6 @@
+{
+    "V1": "layer1.0",
+    "V2": "layer3.1",
+    "V4": "layer3.0",
+    "IT": "layer4.0"
+}

brainscore_vision/models/resnet50_VITO_8deg_cc/requirements.txt

@@ -0,0 +1,3 @@
+torch
+torchvision
+timm

brainscore_vision/models/resnet50_VITO_8deg_cc/test.py

@@ -0,0 +1,8 @@
+import brainscore_vision
+import pytest
+
+
+@pytest.mark.travis_slow
+def test_has_identifier():
+    model = brainscore_vision.load_model('resnet50-VITO-8deg-cc')
+    assert model.identifier == 'resnet50-VITO-8deg-cc'

brainscore_vision/models/resnet50_barlow/__init__.py

@@ -0,0 +1,7 @@
+from brainscore_vision.model_helpers.brain_transformation import ModelCommitment
+from brainscore_vision import model_registry
+from .model import get_layers,get_model
+
+
+model_registry['resnet50-barlow'] = \
+    lambda: ModelCommitment(identifier='resnet50-barlow', activations_model=get_model('resnet50-barlow'), layers=get_layers('resnet50-barlow'))

brainscore_vision/models/resnet50_barlow/model.py

@@ -0,0 +1,53 @@
+import functools
+from brainscore_vision.model_helpers.activations.pytorch import load_preprocess_images
+from brainscore_vision.model_helpers.activations.pytorch import PytorchWrapper
+from brainscore_vision.model_helpers.check_submission import check_models
+import torch
+
+
+
+def get_model(name):
+    """
+    This method fetches an instance of a base model. The instance has to be callable and return a xarray object,
+    containing activations. There exist standard wrapper implementations for common libraries, like pytorch and
+    keras. Checkout the examples folder, to see more. For custom implementations check out the implementation of the
+    wrappers.
+    :param name: the name of the model to fetch
+    :return: the model instance
+    """
+    assert name == 'resnet50-barlow'
+
+    model = torch.hub.load('facebookresearch/barlowtwins:main', 'resnet50')
+    preprocessing = functools.partial(load_preprocess_images, image_size=224)
+    wrapper = PytorchWrapper(identifier='name', model=model, preprocessing=preprocessing)
+    wrapper.image_size = 224
+    return wrapper
+
+
+def get_layers(name):
+    assert name == 'resnet50-barlow'
+    layers = ['conv1', 'layer1.0', 'layer1.1', 'layer1.2',
+        'layer2.0', 'layer2.1', 'layer2.2', 'layer2.3',
+        'layer3.0', 'layer3.1', 'layer3.2', 'layer3.3',
+         'layer3.4', 'layer3.5', 'layer4.0', 'layer4.1', 'layer4.2','avgpool']
+    return layers
+
+
+def get_bibtex(name):
+    """
+    A method returning the bibtex reference of the requested model as a string.
+    """
+    return '''
+@article{zbontar2021barlow,
+  title={Barlow Twins: Self-Supervised Learning via Redundancy Reduction},
+  author={Zbontar, Jure and Jing, Li and Misra, Ishan and LeCun, Yann and Deny, St{\'e}phane},
+  journal={arXiv preprint arXiv:2103.03230},
+  year={2021}
+}
+'''
+
+
+if __name__ == '__main__':
+    # Use this method to ensure the correctness of the BaseModel implementations.
+    # It executes a mock run of brain-score benchmarks.
+    check_models.check_base_models(__name__)

brainscore_vision/models/resnet50_barlow/region_layer_map/resnet50-barlow.json

@@ -0,0 +1 @@
+{"IT": "layer4.0", "V4": "layer2.1", "V1": "layer2.2", "V2": "layer3.0"}

brainscore_vision/models/resnet50_barlow/requirements.txt

@@ -0,0 +1 @@
+torch

brainscore_vision/models/resnet50_barlow/test.py

@@ -0,0 +1,9 @@
+import brainscore_vision
+import pytest
+
+
+
+@pytest.mark.travis_slow
+def test_has_identifier():
+    model = brainscore_vision.load_model('resnet50-barlow')
+    assert model.identifier == 'resnet50-barlow'

brainscore_vision/models/resnet50_finetune_cutmix_AVGe2e3_robust_linf8255_e0_247x234/__init__.py

@@ -0,0 +1,6 @@
+from brainscore_vision.model_helpers.brain_transformation import ModelCommitment
+from brainscore_vision import model_registry
+from .model import get_layers,get_model
+
+model_registry['resnet50_finetune_cutmix_AVGe2e3_robust_linf8255_e0_247x234'] = \
+    lambda: ModelCommitment(identifier='resnet50_finetune_cutmix_AVGe2e3_robust_linf8255_e0_247x234', activations_model=get_model('resnet50_finetune_cutmix_AVGe2e3_robust_linf8255_e0_247x234'), layers=get_layers('resnet50_finetune_cutmix_AVGe2e3_robust_linf8255_e0_247x234'))

brainscore_vision/models/resnet50_finetune_cutmix_AVGe2e3_robust_linf8255_e0_247x234/model.py

@@ -0,0 +1,128 @@
+import functools
+import torch
+from brainscore_vision.model_helpers.activations.pytorch import PytorchWrapper
+from PIL import Image
+import numpy as np
+import timm
+
+import torch.nn as nn
+from albumentations import (
+    Compose, Normalize, Resize,CenterCrop
+    )
+from albumentations.pytorch import ToTensorV2
+
+from brainscore_vision.model_helpers.check_submission import check_models
+from brainscore_vision.model_helpers.s3 import load_weight_file
+from collections import OrderedDict
+
+image_resize = 247
+image_crop = 234
+norm_mean = [0.485, 0.456, 0.406] 
+norm_std = [0.229, 0.224, 0.225]
+
+def custom_image_preprocess(images, **kwargs):
+    
+    transforms_val = Compose([
+        Resize(image_resize, image_resize),
+        CenterCrop(image_crop, image_crop),
+        Normalize(mean=norm_mean,std=norm_std,),
+        ToTensorV2()])
+    
+    images = [np.array(pillow_image) for pillow_image in images]
+    images = [transforms_val(image=image)["image"] for image in images]
+    images = np.stack(images)
+
+    return images
+
+def load_preprocess_images_custom(image_filepaths, preprocess_images=custom_image_preprocess,  **kwargs):
+    images = [load_image(image_filepath) for image_filepath in image_filepaths]
+    images = preprocess_images(images, **kwargs)
+    return images
+
+def load_image(image_filepath):
+    with Image.open(image_filepath) as pil_image:
+        if 'L' not in pil_image.mode.upper() and 'A' not in pil_image.mode.upper()\
+                and 'P' not in pil_image.mode.upper():  # not binary and not alpha and not palletized
+            # work around to https://github.com/python-pillow/Pillow/issues/1144,
+            # see https://stackoverflow.com/a/30376272/2225200
+            return pil_image.copy()
+        else:  # make sure potential binary images are in RGB
+            rgb_image = Image.new("RGB", pil_image.size)
+            rgb_image.paste(pil_image)
+            return rgb_image
+
+class EffNetBX(nn.Module):
+    def __init__(self,):
+        super().__init__ ()
+        self.efnet_model = timm.create_model('resnet50', pretrained=True)
+
+    def forward(self, x):
+        x = self.efnet_model(x)
+        return x
+
+
+def get_model(name):
+    assert name == 'resnet50_finetune_cutmix_AVGe2e3_robust_linf8255_e0_247x234'
+    model= EffNetBX()
+
+    weights_path = load_weight_file(bucket="brainscore-storage", folder_name="brainscore-vision/models",
+                                    relative_path="test_robust_sub/resnet50_robust_cutmixfrontpatchres_e2e3.pth",
+                                    version_id="null",
+                                    sha1="26c09e6ccac6d98bb8d384da35ed59da8f2ffa66")
+    state_dict = torch.load(weights_path, map_location=torch.device('cpu'))["model"]
+
+    new_state_dict = OrderedDict()
+    for k, v in state_dict.items():
+        # strip `module.` prefix
+        if "efnet" in k and not "attacker" in k:
+            name = k.replace("module.model.", "")
+            new_state_dict[name] = v
+
+    res = model.load_state_dict(new_state_dict)
+    model = model.efnet_model
+
+    filter_elems = set(["se", "act", "bn", "conv"])
+    layer_list = [layer for layer, _ in model.named_modules() if not any(i in layer for i in filter_elems)]
+    
+    for n, m in model.named_modules():
+      if isinstance(m, nn.BatchNorm2d):# and any(x in n for x in ["conv_stem" ] + freeze_layers) or n =="bn1":
+        print(f"Freeze {n, m}")
+        m.eval()
+    
+    
+    preprocessing = functools.partial(load_preprocess_images_custom, 
+                                        preprocess_images=custom_image_preprocess,
+                                        )
+
+
+    wrapper = PytorchWrapper(identifier='my-model', model=model, preprocessing=preprocessing, batch_size=8)
+
+    wrapper.image_size = image_crop
+    return wrapper
+
+
+def get_layers(name):
+    assert name == 'resnet50_finetune_cutmix_AVGe2e3_robust_linf8255_e0_247x234'
+    return ['conv1', 
+            #'layer1.0', 'layer1.1','layer1.2',
+            'layer2.0.downsample', 'layer2.0.downsample.0', 'layer2.0.downsample.1', 
+            'layer2.1', 'layer2.2', 'layer2.3', 'layer3', 'layer3.0', 'layer3.0.downsample', 
+            'layer3.0.downsample.0', 'layer3.0.downsample.1', 'layer3.1', 'layer3.2', 'layer3.3', 'layer3.4', 'layer3.5', 
+            'layer4', 'layer4.0', 'layer4.0.downsample', 'layer4.0.downsample.0', 'layer4.0.downsample.1', 'layer4.1', 
+            'layer4.2', 'global_pool', 'global_pool.flatten', 'global_pool.pool',]
+
+    
+
+def get_bibtex(model_identifier):
+    return """@misc{he2015deep,
+      title={Deep Residual Learning for Image Recognition}, 
+      author={Kaiming He and Xiangyu Zhang and Shaoqing Ren and Jian Sun},
+      year={2015},
+      eprint={1512.03385},
+      archivePrefix={arXiv},
+      primaryClass={cs.CV}
+}"""
+
+
+if __name__ == '__main__':
+    check_models.check_base_models(__name__)

brainscore_vision/models/resnet50_finetune_cutmix_AVGe2e3_robust_linf8255_e0_247x234/region_layer_map/resnet50_finetune_cutmix_AVGe2e3_robust_linf8255_e0_247x234.json

@@ -0,0 +1 @@
+{"IT": "layer4.0", "V1": "layer2.2", "V2": "layer3.0", "V4": "layer3.0"}

brainscore_vision/models/resnet50_finetune_cutmix_AVGe2e3_robust_linf8255_e0_247x234/requirements.txt

@@ -0,0 +1,5 @@
+albumentations
+timm
+torch
+PIL
+collections

brainscore_vision/models/resnet50_finetune_cutmix_AVGe2e3_robust_linf8255_e0_247x234/test.py

@@ -0,0 +1,7 @@
+import brainscore_vision
+import pytest
+
+@pytest.mark.travis_slow
+def test_has_identifier():
+    model = brainscore_vision.load_model('resnet50_finetune_cutmix_AVGe2e3_robust_linf8255_e0_247x234')
+    assert model.identifier == 'resnet50_finetune_cutmix_AVGe2e3_robust_linf8255_e0_247x234'

brainscore_vision/models/resnet50_moclr8deg/__init__.py

@@ -0,0 +1,11 @@
+from brainscore_vision.model_helpers.brain_transformation import ModelCommitment
+from brainscore_vision import model_registry
+from .model import get_layers,get_model
+
+
+model_registry['resnet50-moclr8deg'] = \
+    lambda: ModelCommitment(identifier='resnet50-moclr8deg', activations_model=get_model('resnet50-moclr8deg'), layers=get_layers('resnet50-moclr8deg'))
+
+
+
+