brainscore-vision 2.1__py3-none-any.whl

brainscore_vision/models/temporal_model_torchvision/model.py

@@ -0,0 +1,92 @@
+import torch
+import numpy as np
+from torchvision import transforms
+from torchvision.models import video as vid
+
+from brainscore_vision.model_helpers.activations.temporal.model.pytorch import PytorchWrapper
+
+
+LARGE_MODEL_LAYER_STEP = 2
+
+def get_transform_video(transform_img):
+    def transform_video(video):
+        frames = video.to_numpy() / 255.
+        frames = torch.Tensor(frames)
+        frames = frames.permute(0, 3, 1, 2)
+        frames = transform_img(frames)
+        return frames.permute(1, 0, 2, 3)
+    return transform_video
+
+
+def get_model(identifier):
+    if identifier in ["r3d_18", "r2plus1d_18", "mc3_18"]:
+        img_transform = transforms.Compose([
+            transforms.Resize((128, 171)),
+            transforms.CenterCrop(112),
+            transforms.Normalize(mean=[0.43216, 0.394666, 0.37645], std=[0.22803, 0.22145, 0.216989])
+        ])
+        inferencer_kwargs = {
+            "fps": 25,
+            "layer_activation_format": 
+            {
+                "stem": "CTHW",
+                **{f'layer{i}': "CTHW" for i in range(1, 5)},
+                "avgpool": "CTHW",
+                "fc": "C"
+            },
+        }
+        process_output = None
+
+    elif identifier == "s3d":
+        img_transform = transforms.Compose([
+            transforms.Resize((256, 256)),
+            transforms.CenterCrop(224),
+            transforms.Normalize(mean=[0.43216, 0.394666, 0.37645], std=[0.22803, 0.22145, 0.216989])
+        ])
+        inferencer_kwargs = {
+            "fps": 15,
+            "num_frames": (13, np.inf),
+            "layer_activation_format":
+            {
+                **{f"features.{i}": "CTHW" for i in range(0, 16, LARGE_MODEL_LAYER_STEP)},
+                "avgpool": "CTHW",
+                "classifier": "CTHW"
+            }
+        }
+        process_output = None
+
+    elif identifier in ["mvit_v1_b", "mvit_v2_s"]:
+        img_transform = transforms.Compose([
+            transforms.Resize((256, 256)),
+            transforms.CenterCrop(224),
+            transforms.Normalize(mean=[0.43216, 0.394666, 0.37645], std=[0.22803, 0.22145, 0.216989])
+        ])
+        inferencer_kwargs = {
+            "fps": 7.5,
+            "num_frames": 16,
+            "layer_activation_format": {
+                "conv_proj": "CTHW",
+                **{f"blocks.{i}": "THWC" for i in range(0, 16, LARGE_MODEL_LAYER_STEP)},
+                "head": "C",
+            }
+        }
+
+        def process_output(layer, layer_name, input, output):
+            if layer_name.startswith("blocks"):
+                output, thw = output
+                t, h, w = thw
+                output = output[:, 1:]  # remove cls 
+                b, n, c = output.shape
+                assert n == t*h*w
+                output = output.view(b, t, h, w, c)
+                return output
+            return output
+
+    vid_transform = get_transform_video(img_transform)
+    model_name = identifier
+    model = getattr(vid, model_name)(weights="KINETICS400_V1")
+    wrapper = PytorchWrapper(identifier, model, vid_transform, 
+                             process_output=process_output,
+                             **inferencer_kwargs)
+    
+    return wrapper

brainscore_vision/models/temporal_model_torchvision/requirements.txt

@@ -0,0 +1,2 @@
+torch
+torchvision

brainscore_vision/models/temporal_model_torchvision/test.py

@@ -0,0 +1,20 @@
+import pytest
+
+from brainscore_vision import load_model
+
+
+model_list = [
+    "r3d_18",
+    "r2plus1d_18",
+    "mc3_18",
+    "s3d",
+    "mvit_v1_b",
+    "mvit_v2_s",
+]
+
+@pytest.mark.private_access
+@pytest.mark.memory_intense
+@pytest.mark.parametrize("model_identifier", model_list)
+def test_load(model_identifier):
+    model = load_model(model_identifier)
+    assert model is not None

brainscore_vision/models/tv_efficientnet_b1/__init__.py

@@ -0,0 +1,5 @@
+from brainscore_vision import model_registry
+from brainscore_vision.model_helpers.brain_transformation import ModelCommitment
+from .model import get_model, get_layers
+
+model_registry['tv_efficientnet-b1'] = lambda: ModelCommitment(identifier='tv_efficientnet-b1', activations_model=get_model('tv_efficientnet-b1'), layers=get_layers('tv_efficientnet-b1'))

brainscore_vision/models/tv_efficientnet_b1/model.py

@@ -0,0 +1,54 @@
+from brainscore_vision.model_helpers.check_submission import check_models
+import functools
+import os
+import torchvision.models
+from brainscore_vision.model_helpers.activations.pytorch import PytorchWrapper
+from brainscore_vision.model_helpers.activations.pytorch import load_preprocess_images
+from pathlib import Path
+from brainscore_vision.model_helpers import download_weights
+import torch
+
+# This is an example implementation for submitting resnet-50 as a pytorch model
+
+# Attention: It is important, that the wrapper identifier is unique per model!
+# The results will otherwise be the same due to brain-scores internal result caching mechanism.
+# Please load your pytorch model for usage in CPU. There won't be GPUs available for scoring your model.
+# If the model requires a GPU, contact the brain-score team directly.
+from brainscore_vision.model_helpers.check_submission import check_models
+
+
+def get_model_list():
+    return ['tv_efficientnet-b1']
+
+
+def get_model(name):
+    assert name == 'tv_efficientnet-b1'
+    weights = torchvision.models.EfficientNet_B1_Weights.DEFAULT
+    model = torchvision.models.efficientnet_b1(weights=weights)
+    # download_weights(
+    # bucket='brainscore-vision', 
+    # folder_path='models/resnet50-deepdive-2',
+    # filename_version_sha=[('ckpt.pth', 'C722T4BityNPpazdXWiAeu8pGBxKIudb', '0186929df5d04451995d94cd332a3603a00594fe')],
+    # save_directory=Path(__file__).parent)
+    # ckpt_path = os.path.join(os.path.dirname(__file__), 'ckpt.pth')
+    # state_dict = torch.load(ckpt_path, map_location=torch.device('cpu'))
+    # model.load_state_dict(state_dict)
+    
+    preprocessing = functools.partial(load_preprocess_images, image_size=224)
+    print(preprocessing)
+    wrapper = PytorchWrapper(identifier='tv_efficient-b1', model=model, preprocessing=preprocessing)
+    wrapper.image_size = 224
+    return wrapper
+
+
+def get_layers(name):
+    assert name == 'tv_efficientnet-b1'
+    return ['features.1.0.block', 'features.1.1.block', 'features.2.0.block', 'features.2.1.block', 'features.2.2.block','features.3.0.block', 'features.3.1.block', 'features.3.2.block', 'features.4.0.block', 'features.4.1.block', 'features.4.2.block', 'features.4.3.block', 'features.5.0.block', 'features.5.1.block', 'features.5.2.block', 'features.5.3.block',  'features.6.0.block', 'features.6.1.block', 'features.6.2.block', 'features.6.3.block', 'features.6.4.block', 'features.7.0.block', 'features.7.1.block']  
+
+
+def get_bibtex(model_identifier):
+    return """xx"""
+
+
+if __name__ == '__main__':
+    check_models.check_base_models(__name__)

brainscore_vision/models/tv_efficientnet_b1/setup.py

@@ -0,0 +1,24 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from setuptools import setup, find_packages
+
+requirements = [ "torchvision",
+                 "torch"
+]
+
+setup(
+    packages=find_packages(exclude=['tests']),
+    include_package_data=True,
+    install_requires=requirements,
+    license="MIT license",
+    zip_safe=False,
+    keywords='brain-score template',
+    classifiers=[
+        'Development Status :: 2 - Pre-Alpha',
+        'Intended Audience :: Developers',
+        'License :: OSI Approved :: MIT License',
+        'Natural Language :: English',
+    ],
+    test_suite='tests',
+)

brainscore_vision/models/tv_efficientnet_b1/test.py

@@ -0,0 +1 @@
+# Left empty as part of 2023 models migration

brainscore_vision/models/voneresnet_50_non_stochastic/__init__.py

@@ -0,0 +1,7 @@
+from brainscore_vision import model_registry
+from brainscore_vision.model_helpers.brain_transformation import ModelCommitment
+from .model import get_model, get_layers
+
+model_registry['voneresnet-50-non_stochastic'] = lambda: ModelCommitment(identifier='voneresnet-50-non_stochastic',
+                                                               activations_model=get_model('voneresnet-50-non_stochastic'),
+                                                               layers=get_layers('voneresnet-50-non_stochastic'))

brainscore_vision/models/voneresnet_50_non_stochastic/model.py

@@ -0,0 +1,104 @@
+import functools
+from brainscore_vision.model_helpers.activations.pytorch import PytorchWrapper
+from brainscore_vision.model_helpers.check_submission import check_models
+import torch
+from torch.nn import Module
+import torch.nn as nn
+from brainscore_vision.model_helpers.s3 import load_weight_file
+from brainscore_vision.model_helpers.activations.pytorch import load_preprocess_images
+import ssl
+
+# needed import for globals() to work!
+from .vonenet.vonenet.vonenet import VOneNet
+
+ssl._create_default_https_context = ssl._create_unverified_context
+
+
+class Wrapper(Module):
+    def __init__(self, model):
+        super(Wrapper, self).__init__()
+        self.module = model
+
+
+def get_model_from_s3():
+    model_arch = 'resnet50'
+    pretrained = True
+    if pretrained and model_arch:
+        weights_path = load_weight_file(bucket="brainscore-vision", folder_name="models",
+                                        relative_path="voneresnet-50-non_stochastic/voneresnet50_ns_e70.pth.tar",
+                                        version_id="vDk2cwi2xjwGqhGyyjp8lEGSfcaFzB61",
+                                        sha1="c270528818d6d7fc67a6aec86919d47311ad6221")
+        ckpt_data = torch.load(weights_path, map_location=torch.device('cpu'))
+        stride = ckpt_data['flags']['stride']
+        simple_channels = ckpt_data['flags']['simple_channels']
+        complex_channels = ckpt_data['flags']['complex_channels']
+        k_exc = ckpt_data['flags']['k_exc']
+
+        noise_mode = ckpt_data['flags']['noise_mode']
+        noise_scale = ckpt_data['flags']['noise_scale']
+        noise_level = ckpt_data['flags']['noise_level']
+
+        model_id = ckpt_data['flags']['arch'].replace('_', '').lower()
+
+        model = globals()[f'VOneNet'](model_arch=model_id, stride=stride, k_exc=k_exc,
+                                      simple_channels=simple_channels, complex_channels=complex_channels,
+                                      noise_mode=noise_mode, noise_scale=noise_scale, noise_level=noise_level)
+
+        if model_arch.lower() == 'resnet50_at':
+            ckpt_data['state_dict'].pop('vone_block.div_u.weight')
+            ckpt_data['state_dict'].pop('vone_block.div_t.weight')
+            model.load_state_dict(ckpt_data['state_dict'])
+        else:
+            model = Wrapper(model)
+            model.load_state_dict(ckpt_data['state_dict'])
+            model = model.module
+
+        model = nn.DataParallel(model)
+    else:
+        model = globals()[f'VOneNet'](model_arch=model_arch)
+        model = nn.DataParallel(model)
+
+    model.to("cpu")
+    return model
+
+def get_model(name):
+    assert name == 'voneresnet-50-non_stochastic'
+    model = get_model_from_s3()
+    model = model.module
+    preprocessing = functools.partial(load_preprocess_images, image_size=224,
+                                      normalize_mean=(0.5, 0.5, 0.5), normalize_std=(0.5, 0.5, 0.5))
+    wrapper = PytorchWrapper(identifier='vone' + 'resnet50_ns', model=model, preprocessing=preprocessing)
+    wrapper.image_size = 224
+    return wrapper
+
+
+def get_layers(name):
+    assert name == 'voneresnet-50-non_stochastic'
+    layers = (
+            ['vone_block'] +
+            ['model.layer1.0', 'model.layer1.1', 'model.layer1.2'] +
+            ['model.layer2.0', 'model.layer2.1', 'model.layer2.2', 'model.layer2.3'] +
+            ['model.layer3.0', 'model.layer3.1', 'model.layer3.2', 'model.layer3.3',
+             'model.layer3.4', 'model.layer3.5'] +
+            ['model.layer4.0', 'model.layer4.1', 'model.layer4.2'] +
+            ['model.avgpool']
+    )
+    return layers
+
+
+def get_bibtex(model_identifier):
+    return """@inproceedings{NEURIPS2020_98b17f06,
+ author = {Dapello, Joel and Marques, Tiago and Schrimpf, Martin and Geiger, Franziska and Cox, David and DiCarlo, James J},
+ booktitle = {Advances in Neural Information Processing Systems},
+ editor = {H. Larochelle and M. Ranzato and R. Hadsell and M.F. Balcan and H. Lin},
+ pages = {13073--13087},
+ publisher = {Curran Associates, Inc.},
+ title = {Simulating a Primary Visual Cortex at the Front of CNNs Improves Robustness to Image Perturbations},
+ url = {https://proceedings.neurips.cc/paper_files/paper/2020/file/98b17f068d5d9b7668e19fb8ae470841-Paper.pdf},
+ volume = {33},
+ year = {2020}
+}"""
+
+
+if __name__ == '__main__':
+    check_models.check_base_models(__name__)

brainscore_vision/models/voneresnet_50_non_stochastic/requirements.txt

@@ -0,0 +1,8 @@
+torch
+torchvision
+numpy
+pandas
+scipy
+tqdm
+fire
+requests

brainscore_vision/models/voneresnet_50_non_stochastic/test.py

@@ -0,0 +1,8 @@
+import pytest
+import brainscore_vision
+
+
+@pytest.mark.travis_slow
+def test_has_identifier():
+    model = brainscore_vision.load_model('voneresnet-50-non_stochastic')
+    assert model.identifier == 'voneresnet-50-non_stochastic'