brainscore-vision 2.1__py3-none-any.whl

tests/test_model_helpers/activations/test___init__.py

@@ -0,0 +1,404 @@
+import functools
+import numpy as np
+import os
+import pytest
+import xarray as xr
+from pathlib import Path
+
+from brainio.stimuli import StimulusSet
+from brainscore_vision.model_helpers.activations import PytorchWrapper
+from brainscore_vision.model_helpers.activations.core import flatten
+from brainscore_vision.model_helpers.activations.pca import LayerPCA
+
+
+def unique_preserved_order(a):
+    _, idx = np.unique(a, return_index=True)
+    return a[np.sort(idx)]
+
+
+def pytorch_custom():
+    import torch
+    from torch import nn
+    from brainscore_vision.model_helpers.activations.pytorch import load_preprocess_images
+
+    class MyModel(nn.Module):
+        def __init__(self):
+            super(MyModel, self).__init__()
+            self.conv1 = torch.nn.Conv2d(in_channels=3, out_channels=2, kernel_size=3)
+            self.relu1 = torch.nn.ReLU()
+            linear_input_size = np.power((224 - 3 + 2 * 0) / 1 + 1, 2) * 2
+            self.linear = torch.nn.Linear(int(linear_input_size), 1000)
+            self.relu2 = torch.nn.ReLU()  # can't get named ReLU output otherwise
+
+        def forward(self, x):
+            x = self.conv1(x)
+            x = self.relu1(x)
+            x = x.view(x.size(0), -1)
+            x = self.linear(x)
+            x = self.relu2(x)
+            return x
+
+    preprocessing = functools.partial(load_preprocess_images, image_size=224)
+    return PytorchWrapper(model=MyModel(), preprocessing=preprocessing)
+
+
+def pytorch_alexnet():
+    from torchvision.models.alexnet import alexnet
+    from brainscore_vision.model_helpers.activations.pytorch import load_preprocess_images
+
+    preprocessing = functools.partial(load_preprocess_images, image_size=224)
+    return PytorchWrapper(model=alexnet(pretrained=True), preprocessing=preprocessing)
+
+
+def pytorch_alexnet_resize():
+    from torchvision.models.alexnet import alexnet
+    from brainscore_vision.model_helpers.activations.pytorch import load_images, torchvision_preprocess
+    from torchvision import transforms
+    torchvision_preprocess_input = transforms.Compose([transforms.Resize(224), torchvision_preprocess()])
+
+    def preprocessing(paths):
+        images = load_images(paths)
+        images = [torchvision_preprocess_input(image) for image in images]
+        images = np.concatenate(images)
+        return images
+
+    return PytorchWrapper(alexnet(pretrained=True), preprocessing, identifier='alexnet-resize')
+
+
+def pytorch_transformer_substitute():
+    import torch
+    from torch import nn
+    from brainscore_vision.model_helpers.activations.pytorch import load_preprocess_images
+
+    class MyTransformer(nn.Module):
+        def __init__(self):
+            super(MyTransformer, self).__init__()
+            self.conv = torch.nn.Conv1d(in_channels=3, out_channels=2, kernel_size=3)
+            self.relu1 = torch.nn.ReLU()
+            linear_input_size = (224 ** 2 - 2) * 2
+            self.linear = torch.nn.Linear(int(linear_input_size), 1000)
+            self.relu2 = torch.nn.ReLU()  # logit out needs to be 1000
+
+        def forward(self, x):
+            x = x.view(*x.shape[:2], -1)
+            x = self.conv(x)
+            x = self.relu1(x)
+            x = x.view(x.shape[0], -1)
+            x = self.linear(x)
+            x = self.relu2(x)
+
+            return x
+
+    preprocessing = functools.partial(load_preprocess_images, image_size=224)
+    return PytorchWrapper(model=MyTransformer(), preprocessing=preprocessing)
+
+
+models_layers = [
+    pytest.param(pytorch_custom, ['linear', 'relu2']),
+    pytest.param(pytorch_alexnet, ['features.12', 'classifier.5'], marks=pytest.mark.memory_intense),
+    pytest.param(pytorch_transformer_substitute, ['relu1']),
+]
+
+# exact microsaccades for pytorch_alexnet, grayscale.png, for 1 and 10 number_of_trials
+exact_microsaccades = {"x_degrees": {1: np.array([0.]),
+                                     10: np.array([0., -0.00639121, -0.02114204, -0.02616418, -0.02128906,
+                                                   -0.00941355, 0.00596172, 0.02166913, 0.03523793, 0.04498976])},
+                       "y_degrees": {1: np.array([0.]),
+                                     10: np.array([0., 0.0144621, 0.00728107, -0.00808922, -0.02338324, -0.0340791,
+                                                   -0.03826824, -0.03578336, -0.02753704, -0.01503068])},
+                       "x_pixels": {1: np.array([0.]),
+                                    10: np.array([0., -0.17895397, -0.59197722, -0.73259714, -0.59609364, -0.26357934,
+                                                  0.16692818, 0.60673569, 0.98666196, 1.25971335])},
+                       "y_pixels": {1: np.array([0.]),
+                                    10: np.array([0., 0.40493885, 0.20386999, -0.22649819, -0.65473077, -0.95421482,
+                                                  -1.07151061, -1.00193403, -0.77103707, -0.42085896])}}
+
+
+@pytest.mark.parametrize("image_name", ['rgb.jpg', 'grayscale.png', 'grayscale2.jpg', 'grayscale_alpha.png',
+                                        'palletized.png'])
+@pytest.mark.parametrize(["pca_components", "logits"], [(None, True), (None, False), (5, False)])
+@pytest.mark.parametrize(["model_ctr", "layers"], models_layers)
+def test_from_image_path(model_ctr, layers, image_name, pca_components, logits):
+    stimuli_paths = [os.path.join(os.path.dirname(__file__), image_name)]
+
+    activations_extractor = model_ctr()
+    if pca_components:
+        LayerPCA.hook(activations_extractor, pca_components)
+    activations = activations_extractor.from_paths(stimuli_paths=stimuli_paths,
+                                                   layers=layers if not logits else None)
+
+    assert activations is not None
+    assert len(activations['stimulus_path']) == 1
+    assert len(np.unique(activations['layer'])) == len(layers) if not logits else 1
+    if logits and not pca_components:
+        assert len(activations['neuroid']) == 1000
+    elif pca_components is not None:
+        assert len(activations['neuroid']) == pca_components * len(layers)
+    import gc
+    gc.collect()  # free some memory, we're piling up a lot of activations at this point
+    return activations
+
+@pytest.mark.memory_intense
+@pytest.mark.parametrize("image_name", ['rgb.jpg', 'grayscale.png', 'grayscale2.jpg', 'grayscale_alpha.png',
+                                        'palletized.png'])
+@pytest.mark.parametrize(["model_ctr", "layers"], models_layers)
+@pytest.mark.parametrize("number_of_trials", [1, 3, 10])
+def test_require_variance_has_shift_coords(model_ctr, layers, image_name, number_of_trials):
+    stimulus_paths = [os.path.join(os.path.dirname(__file__), image_name)]
+    activations_extractor = model_ctr()
+    # when using microsaccades, the ModelCommitment sets its visual angle. Since this test skips the ModelCommitment,
+    #  we set it here manually.
+    activations_extractor._extractor.set_visual_degrees(8.)
+
+    activations = activations_extractor(stimuli=stimulus_paths, layers=layers, number_of_trials=number_of_trials,
+                                        require_variance=True)
+
+    assert activations is not None
+    assert len(activations['microsaccade_shift_x_pixels']) == number_of_trials * len(stimulus_paths)
+    assert len(activations['microsaccade_shift_y_pixels']) == number_of_trials * len(stimulus_paths)
+    assert len(activations['microsaccade_shift_x_degrees']) == number_of_trials * len(stimulus_paths)
+    assert len(activations['microsaccade_shift_y_degrees']) == number_of_trials * len(stimulus_paths)
+
+@pytest.mark.memory_intense
+@pytest.mark.parametrize("image_name", ['rgb.jpg', 'grayscale.png', 'grayscale2.jpg', 'grayscale_alpha.png',
+                                        'palletized.png'])
+@pytest.mark.parametrize(["model_ctr", "layers"], models_layers)
+@pytest.mark.parametrize("require_variance", [False, True])
+@pytest.mark.parametrize("number_of_trials", [1, 3, 10])
+def test_require_variance_presentation_length(model_ctr, layers, image_name, require_variance, number_of_trials):
+    stimulus_paths = [os.path.join(os.path.dirname(__file__), image_name)]
+    activations_extractor = model_ctr()
+    # when using microsaccades, the ModelCommitment sets its visual angle. Since this test skips the ModelCommitment,
+    #  we set it here manually.
+    activations_extractor._extractor.set_visual_degrees(8.)
+
+    activations = activations_extractor(stimuli=stimulus_paths, layers=layers,
+                                        number_of_trials=number_of_trials, require_variance=require_variance)
+
+    assert activations is not None
+    if require_variance:
+        assert len(activations['presentation']) == number_of_trials
+    else:
+        assert len(activations['presentation']) == 1
+
+@pytest.mark.memory_intense
+@pytest.mark.parametrize("image_name", ['rgb.jpg', 'grayscale.png', 'grayscale2.jpg', 'grayscale_alpha.png',
+                                        'palletized.png'])
+@pytest.mark.parametrize(["model_ctr", "layers"], models_layers)
+def test_temporary_file_handling(model_ctr, layers, image_name):
+    import tempfile
+    stimulus_paths = [os.path.join(os.path.dirname(__file__), image_name)]
+    activations_extractor = model_ctr()
+    # when using microsaccades, the ModelCommitment sets its visual angle. Since this test skips the ModelCommitment,
+    #  we set it here manually.
+    activations_extractor._extractor.set_visual_degrees(8.)
+
+    activations = activations_extractor(stimuli=stimulus_paths, layers=layers, number_of_trials=2,
+                                        require_variance=True)
+    temp_files = [f for f in os.listdir(tempfile.gettempdir()) if f.startswith('temp') and f.endswith('.png')]
+
+    assert activations is not None
+    assert len(temp_files) == 0
+
+
+def _build_stimulus_set(image_names):
+    stimulus_set = StimulusSet([{'stimulus_id': image_name, 'some_meta': image_name[::-1]}
+                                for image_name in image_names])
+    stimulus_set.stimulus_paths = {image_name: os.path.join(os.path.dirname(__file__), image_name)
+                                   for image_name in image_names}
+    return stimulus_set
+
+
+@pytest.mark.parametrize("pca_components", [None, 5])
+@pytest.mark.parametrize(["model_ctr", "layers"], models_layers)
+def test_from_stimulus_set(model_ctr, layers, pca_components):
+    image_names = ['rgb.jpg', 'grayscale.png', 'grayscale2.jpg', 'grayscale_alpha.png', 'palletized.png']
+    stimulus_set = _build_stimulus_set(image_names)
+
+    activations_extractor = model_ctr()
+    if pca_components:
+        LayerPCA.hook(activations_extractor, pca_components)
+    activations = activations_extractor.from_stimulus_set(stimulus_set, layers=layers, stimuli_identifier=False)
+
+    assert activations is not None
+    assert set(activations['stimulus_id'].values) == set(image_names)
+    assert all(activations['some_meta'].values == [image_name[::-1] for image_name in image_names])
+    assert len(np.unique(activations['layer'])) == len(layers)
+    if pca_components is not None:
+        assert len(activations['neuroid']) == pca_components * len(layers)
+
+
+@pytest.mark.memory_intense
+@pytest.mark.parametrize("number_of_trials", [3, 10])
+@pytest.mark.parametrize(["model_ctr", "layers"], models_layers)
+def test_microsaccades_from_stimulus_set(model_ctr, layers, number_of_trials):
+    image_names = ['rgb.jpg', 'grayscale.png', 'grayscale2.jpg', 'grayscale_alpha.png', 'palletized.png']
+    stimulus_set = _build_stimulus_set(image_names)
+
+    activations_extractor = model_ctr()
+    activations_extractor._extractor.set_visual_degrees(8.)
+    activations_extractor._extractor._microsaccade_helper.number_of_trials = number_of_trials
+    activations = activations_extractor.from_stimulus_set(stimulus_set, layers=layers, stimuli_identifier=False,
+                                                          require_variance=True)
+
+    assert activations is not None
+    assert len(activations['presentation']) == len(image_names) * number_of_trials
+    assert set(activations['stimulus_id'].values) == set(image_names)
+    assert len(np.unique(activations['layer'])) == len(layers)
+
+
+@pytest.mark.memory_intense
+@pytest.mark.parametrize("pca_components", [None, 1000])
+def test_exact_activations(pca_components):
+    activations = test_from_image_path(model_ctr=pytorch_alexnet_resize, layers=['features.12', 'classifier.5'],
+                                       image_name='rgb.jpg', pca_components=pca_components, logits=False)
+    path_to_expected = Path(__file__).parent / f'alexnet-rgb-{pca_components}.nc'
+    expected = xr.load_dataarray(path_to_expected)
+
+    # Originally, the `stimulus_path` Index was used to index into xarrays in Brain-Score, but this was changed
+    #  as a part of PR #492 to a MultiIndex to allow metadata to be attached to multiple repetitions of the same
+    #  `stimulus_path`. Old .nc files need to be updated to use the `presentation` index instead of `stimulus_path`,
+    #  and instead of changing the extant activations, this test was simply modified to simulate that.
+    expected = expected.rename({'stimulus_path': 'presentation'})
+
+    assert (activations == expected).all()
+
+
+@pytest.mark.memory_intense
+@pytest.mark.parametrize("number_of_trials", [1, 10])
+def test_exact_microsaccades(number_of_trials):
+    image_name = 'grayscale.png'
+    stimulus_paths = [os.path.join(os.path.dirname(__file__), image_name)]
+    activations_extractor = pytorch_alexnet()
+    # when using microsaccades, the ModelCommitment sets its visual angle. Since this test skips the ModelCommitment,
+    #  we set it here manually.
+    activations_extractor._extractor.set_visual_degrees(8.)
+    # the exact microsaccades were computed at this extent
+    assert activations_extractor._extractor._microsaccade_helper.microsaccade_extent_degrees == 0.05
+
+    activations = activations_extractor(stimuli=stimulus_paths, layers=['features.12'],
+                                        number_of_trials=number_of_trials, require_variance=True)
+
+    assert activations is not None
+    # test with np.isclose instead of == since while the arrays are visually equal, == often fails due to float errors
+    assert np.isclose(activations['microsaccade_shift_x_degrees'].values,
+                      exact_microsaccades['x_degrees'][number_of_trials],
+                      rtol=1e-05,
+                      atol=1e-08).all()
+    assert np.isclose(activations['microsaccade_shift_y_degrees'].values,
+                       exact_microsaccades['y_degrees'][number_of_trials],
+                       rtol=1e-05,
+                       atol=1e-08).all()
+    assert np.isclose(activations['microsaccade_shift_x_pixels'].values,
+                      exact_microsaccades['x_pixels'][number_of_trials],
+                      rtol=1e-05,
+                      atol=1e-08).all()
+    assert np.isclose(activations['microsaccade_shift_y_pixels'].values,
+                      exact_microsaccades['y_pixels'][number_of_trials],
+                      rtol=1e-05,
+                      atol=1e-08).all()
+
+
+@pytest.mark.memory_intense
+@pytest.mark.parametrize(["model_ctr", "internal_layers"], [
+    (pytorch_alexnet, ['features.12', 'classifier.5']),
+])
+def test_mixed_layer_logits(model_ctr, internal_layers):
+    stimuli_paths = [os.path.join(os.path.dirname(__file__), 'rgb.jpg')]
+
+    activations_extractor = model_ctr()
+    layers = internal_layers + ['logits']
+    activations = activations_extractor(stimuli=stimuli_paths, layers=layers)
+    assert len(np.unique(activations['layer'])) == len(internal_layers) + 1
+    assert set(activations['layer'].values) == set(layers)
+    assert unique_preserved_order(activations['layer'])[-1] == 'logits'
+
+
+@pytest.mark.memory_intense
+@pytest.mark.parametrize(["model_ctr", "expected_identifier"], [
+    (pytorch_custom, 'MyModel'),
+    (pytorch_alexnet, 'AlexNet'),
+])
+def test_infer_identifier(model_ctr, expected_identifier):
+    model = model_ctr()
+    assert model.identifier == expected_identifier
+
+
+def test_transformer_meta():
+    model = pytorch_transformer_substitute()
+    activations = model(stimuli=[os.path.join(os.path.dirname(__file__), 'rgb.jpg')], layers=['relu1'])
+    assert hasattr(activations, 'channel')
+    assert hasattr(activations, 'embedding')
+    assert len(set(activations['neuroid_id'].values)) == len(activations['neuroid'])
+
+
+def test_convolution_meta():
+    model = pytorch_custom()
+    activations = model(stimuli=[os.path.join(os.path.dirname(__file__), 'rgb.jpg')], layers=['conv1'])
+    assert hasattr(activations, 'channel')
+    assert hasattr(activations, 'channel_x')
+    assert hasattr(activations, 'channel_y')
+    assert len(set(activations['neuroid_id'].values)) == len(activations['neuroid'])
+
+
+def test_conv_and_fc():
+    model = pytorch_custom()
+    activations = model(stimuli=[os.path.join(os.path.dirname(__file__), 'rgb.jpg')], layers=['conv1', 'linear'])
+    assert set(activations['layer'].values) == {'conv1', 'linear'}
+
+
+@pytest.mark.timeout(300)
+def test_merge_large_layers():
+    import torch
+    from torch import nn
+    from brainscore_vision.model_helpers.activations.pytorch import load_preprocess_images
+
+    class LargeModel(nn.Module):
+        def __init__(self):
+            super(LargeModel, self).__init__()
+            self.conv = torch.nn.Conv2d(in_channels=3, out_channels=4, kernel_size=3)
+            self.relu = torch.nn.ReLU()
+
+        def forward(self, x):
+            x = self.conv(x)
+            x = self.relu(x)
+            return x
+
+    preprocessing = functools.partial(load_preprocess_images, image_size=224)
+    model = PytorchWrapper(model=LargeModel(), preprocessing=preprocessing)
+    activations = model(stimuli=[os.path.join(os.path.dirname(__file__), 'rgb.jpg')] * 64, layers=['conv', 'relu'])
+    assert len(activations['neuroid']) == 394272
+    assert len(set(activations['neuroid_id'].values)) == len(activations['neuroid'])
+    assert set(activations['layer'].values) == {'conv', 'relu'}
+
+
+class TestFlatten:
+    def test_flattened_shape(self):
+        A = np.random.rand(2560, 256, 6, 6)
+        flattened = flatten(A)
+        assert np.prod(flattened.shape) == np.prod(A.shape)
+        assert flattened.shape[0] == A.shape[0]
+        assert len(flattened.shape) == 2
+
+    def test_indices_shape(self):
+        A = np.random.rand(2560, 256, 6, 6)
+        _, indices = flatten(A, return_index=True)
+        assert len(indices.shape) == 2
+        assert indices.shape[0] == np.prod(A.shape[1:])
+        assert indices.shape[1] == 3  # for 256, 6, 6
+
+    def test_match_flatten(self):
+        A = np.random.rand(10, 256, 6, 6)
+        flattened, indices = flatten(A, return_index=True)
+        for layer in range(A.shape[0]):
+            for i in range(np.prod(A.shape[1:])):
+                value = flattened[layer][i]
+                index = indices[i]
+                assert A[layer][tuple(index)] == value
+
+    def test_inverse(self):
+        A = np.random.rand(2560, 256, 6, 6)
+        flattened = flatten(A)
+        A_ = np.reshape(flattened, [flattened.shape[0], 256, 6, 6])
+        assert A.shape == A_.shape
+        assert (A == A_).all()

tests/test_model_helpers/brain_transformation/test___init__.py

@@ -0,0 +1,18 @@
+from unittest.mock import Mock
+
+from brainscore_vision.model_helpers.brain_transformation import ModelCommitment
+from brainscore_vision.model_helpers.utils import fullname
+
+
+class TestVisualDegrees:
+    def test_standard_commitment(self):
+        # create mock ActivationsExtractorHelper with a mock set_visual_degrees to avoid failing set_visual_degrees()
+        mock_extractor = Mock()
+        mock_extractor.set_visual_degrees = Mock()
+        mock_activations_model = Mock()
+        mock_activations_model._extractor = mock_extractor
+
+        # Initialize ModelCommitment with the mock activations_model
+        brain_model = ModelCommitment(identifier=fullname(self), activations_model=mock_activations_model,
+                                      layers=['dummy'])
+        assert brain_model.visual_degrees() == 8

tests/test_model_helpers/brain_transformation/test_behavior.py

@@ -0,0 +1,181 @@
+import functools
+import os
+
+import numpy as np
+import pytest
+from pytest import approx
+
+import brainscore_vision
+from brainio.assemblies import BehavioralAssembly
+from brainio.stimuli import StimulusSet
+from brainscore_vision.benchmark_helpers.screen import place_on_screen
+from brainscore_vision.model_helpers.activations import PytorchWrapper
+from brainscore_vision.model_helpers.brain_transformation import ModelCommitment
+from brainscore_vision.model_interface import BrainModel
+
+
+def pytorch_custom():
+    import torch
+    from torch import nn
+    from brainscore_vision.model_helpers.activations.pytorch import load_preprocess_images
+
+    class MyModel(nn.Module):
+        def __init__(self):
+            super(MyModel, self).__init__()
+            np.random.seed(0)
+            torch.random.manual_seed(0)
+            self.conv1 = torch.nn.Conv2d(in_channels=3, out_channels=2, kernel_size=3)
+            self.relu1 = torch.nn.ReLU()
+            linear_input_size = np.power((224 - 3 + 2 * 0) / 1 + 1, 2) * 2
+            self.linear = torch.nn.Linear(int(linear_input_size), 1000)
+            self.relu2 = torch.nn.ReLU()  # can't get named ReLU output otherwise
+
+        def forward(self, x):
+            x = self.conv1(x)
+            x = self.relu1(x)
+            x = x.view(x.size(0), -1)
+            x = self.linear(x)
+            x = self.relu2(x)
+            return x
+
+    preprocessing = functools.partial(load_preprocess_images, image_size=224)
+    return PytorchWrapper(model=MyModel(), preprocessing=preprocessing)
+
+
+class TestLabelBehavior:
+    @pytest.mark.parametrize(['model_ctr'], [(pytorch_custom,)])
+    def test_imagenet_creates_synset(self, model_ctr):
+        activations_model = model_ctr()
+        brain_model = ModelCommitment(identifier=activations_model.identifier, activations_model=activations_model,
+                                      layers=None, behavioral_readout_layer='dummy')  # not needed
+        stimuli = mock_stimulus_set()
+        brain_model.start_task(BrainModel.Task.label, 'imagenet')
+        behavior = brain_model.look_at(stimuli)
+        assert isinstance(behavior, BehavioralAssembly)
+        assert set(behavior['stimulus_id'].values) == {'1', '2'}
+        assert len(behavior['synset']) == 2
+        assert behavior['synset'].values[0].startswith('n')
+
+    @pytest.mark.parametrize(['model_ctr'], [(pytorch_custom,)])
+    def test_choicelabels(self, model_ctr):
+        activations_model = model_ctr()
+        brain_model = ModelCommitment(identifier=activations_model.identifier, activations_model=activations_model,
+                                      layers=['relu1', 'relu2'], behavioral_readout_layer='relu2')
+        stimuli = mock_stimulus_set()
+        choice_labels = ['dog', 'cat', 'bear', 'bird']
+        brain_model.start_task(BrainModel.Task.label, choice_labels)
+        behavior = brain_model.look_at(stimuli)
+        assert isinstance(behavior, BehavioralAssembly)
+        assert set(behavior['stimulus_id'].values) == {'1', '2'}
+        assert all(choice in choice_labels for choice in behavior.squeeze().values)
+        # these two labels do not necessarily make sense since we're working with a random model
+        assert behavior.sel(stimulus_id='1').values.item() == 'bear'
+        assert behavior.sel(stimulus_id='2').values.item() == 'bird'
+
+
+def mock_stimulus_set():
+    stimuli = StimulusSet({'stimulus_id': ['1', '2'], 'filename': ['rgb1', 'rgb2']})
+    stimuli.stimulus_paths = {'1': os.path.join(os.path.dirname(__file__), 'rgb1.jpg'),
+                              '2': os.path.join(os.path.dirname(__file__), 'rgb2.jpg')}
+    stimuli.identifier = 'TestLabelBehavior.rgb_1_2'
+    return stimuli
+
+def mock_triplet():
+    stimuli = StimulusSet({'stimulus_id': ['1', '2', '3'], 'filename': ['rgb1', 'rgb2', 'rgb3']})
+    stimuli.stimulus_paths = {'1': os.path.join(os.path.dirname(__file__), 'rgb1.jpg'),
+                              '2': os.path.join(os.path.dirname(__file__), 'rgb2.jpg'),
+                              '3': os.path.join(os.path.dirname(__file__), 'rgb3.jpg')}
+    stimuli.identifier = 'TestLabelBehavior.rgb_1_2_3'
+    return stimuli
+
+
+class TestLogitsBehavior:
+    """
+    legacy support for LogitsBehavior; still used in old candidate_models submissions
+    https://github.com/brain-score/candidate_models/blob/fa965c452bd17c6bfcca5b991fdbb55fd5db618f/candidate_models/model_commitments/cornets.py#L13
+    """
+
+    def test_import(self):
+        # noinspection PyUnresolvedReferences
+        from brainscore_vision.model_helpers.brain_transformation.behavior import LogitsBehavior
+
+    def test_imagenet(self):
+        from brainscore_vision.model_helpers.brain_transformation.behavior import LogitsBehavior
+
+        activations_model = pytorch_custom()
+        logits_behavior = LogitsBehavior(
+            identifier='pytorch-custom', activations_model=activations_model)
+
+        stimuli = mock_stimulus_set()
+        logits_behavior.start_task(BrainModel.Task.label, 'imagenet')
+        behavior = logits_behavior.look_at(stimuli)
+        assert isinstance(behavior, BehavioralAssembly)
+        assert set(behavior['stimulus_id'].values) == {'1', '2'}
+        assert len(behavior['synset']) == 2
+        assert behavior['synset'].values[0].startswith('n')
+
+
+class TestProbabilitiesMapping:
+    def test_creates_probabilities(self):
+        activations_model = pytorch_custom()
+        brain_model = ModelCommitment(identifier=activations_model.identifier, activations_model=activations_model,
+                                      layers=None, behavioral_readout_layer='relu2')
+        fitting_stimuli = StimulusSet({'stimulus_id': ['rgb1', 'rgb2'], 'image_label': ['label1', 'label2']})
+        fitting_stimuli.stimulus_paths = {'rgb1': os.path.join(os.path.dirname(__file__), 'rgb1.jpg'),
+                                          'rgb2': os.path.join(os.path.dirname(__file__), 'rgb2.jpg')}
+        fitting_stimuli.identifier = 'test_probabilities_mapping.creates_probabilities'
+        fitting_stimuli = place_on_screen(fitting_stimuli, target_visual_degrees=brain_model.visual_degrees(),
+                                          source_visual_degrees=8)
+        brain_model.start_task(BrainModel.Task.probabilities, fitting_stimuli)
+        probabilities = brain_model.look_at(fitting_stimuli)
+        np.testing.assert_array_equal(probabilities.dims, ['presentation', 'choice'])
+        np.testing.assert_array_equal(probabilities.shape, [2, 2])
+        np.testing.assert_array_almost_equal(probabilities.sel(stimulus_id='rgb1', choice='label1').values,
+                                             probabilities.sel(stimulus_id='rgb2', choice='label2').values)
+        assert probabilities.sel(stimulus_id='rgb1', choice='label1') + \
+               probabilities.sel(stimulus_id='rgb1', choice='label2') == approx(1)
+
+
+class TestOddOneOut:
+    def test_import(self):
+        from brainscore_vision.model_helpers.brain_transformation.behavior import OddOneOut
+
+    def test_dot(self):
+        from brainscore_vision.model_helpers.brain_transformation.behavior import OddOneOut
+    
+        # Set up the task
+        activations_model = pytorch_custom()
+        brain_model = OddOneOut(identifier='pytorch-custom',
+                                activations_model=activations_model,
+                                layer=["relu2"])
+
+        # Test similarity measure functionality
+        assert brain_model.similarity_measure == 'dot'
+
+        # Test the task and output
+        stimuli = mock_triplet()
+        brain_model.start_task(BrainModel.Task.odd_one_out)
+        choice = brain_model.look_at(stimuli)
+        assert isinstance(choice, BehavioralAssembly)
+        assert len(choice.values) == 1
+
+    def test_cosine(self):
+        from brainscore_vision.model_helpers.brain_transformation.behavior import OddOneOut
+
+        # Set up the task
+        activations_model = pytorch_custom()
+        brain_model = OddOneOut(identifier='pytorch-custom',
+                                activations_model=activations_model,
+                                layer=["relu2"])
+        
+        # Test similarity measure functionality
+        brain_model.set_similarity_measure('cosine') 
+        assert brain_model.similarity_measure == 'cosine'
+        
+        # Test the task and output
+        stimuli = mock_triplet()
+        brain_model.start_task(BrainModel.Task.odd_one_out)
+        choice = brain_model.look_at(stimuli)
+        assert isinstance(choice, BehavioralAssembly)
+        assert len(choice.values) == 1
+        

tests/test_model_helpers/brain_transformation/test_neural.py

@@ -0,0 +1,70 @@
+import functools
+import os
+from pathlib import Path
+
+import numpy as np
+import pytest
+
+from brainio.stimuli import StimulusSet
+from brainscore_vision.model_helpers.activations import PytorchWrapper
+from brainscore_vision.model_helpers.brain_transformation import ModelCommitment, LayerMappedModel
+
+
+def pytorch_custom():
+    import torch
+    from torch import nn
+    from brainscore_vision.model_helpers.activations.pytorch import load_preprocess_images
+
+    class MyModel(nn.Module):
+        def __init__(self):
+            super(MyModel, self).__init__()
+            self.conv1 = torch.nn.Conv2d(in_channels=3, out_channels=2, kernel_size=3)
+            self.relu1 = torch.nn.ReLU()
+            linear_input_size = np.power((224 - 3 + 2 * 0) / 1 + 1, 2) * 2
+            self.linear = torch.nn.Linear(int(linear_input_size), 1000)
+            self.relu2 = torch.nn.ReLU()  # can't get named ReLU output otherwise
+
+        def forward(self, x):
+            x = self.conv1(x)
+            x = self.relu1(x)
+            x = x.view(x.size(0), -1)
+            x = self.linear(x)
+            x = self.relu2(x)
+            return x
+
+    preprocessing = functools.partial(load_preprocess_images, image_size=224)
+    return PytorchWrapper(model=MyModel(), preprocessing=preprocessing)
+
+
+@pytest.mark.memory_intense
+class TestLayerSelection:
+    @pytest.mark.parametrize(['model_ctr', 'layers', 'expected_layer', 'region'],
+                             [(pytorch_custom, ['linear', 'relu2'], 'relu2', 'IT')])
+    def test_commit_record(self, model_ctr, layers, expected_layer, region):
+        activations_model = model_ctr()
+        brain_model = ModelCommitment(identifier=activations_model.identifier, activations_model=activations_model,
+                                      layers=layers)
+
+        brain_model.start_recording(region, [(70, 170)])
+        predictions = brain_model.look_at([Path(__file__).parent / 'rgb1.jpg'])
+        assert set(predictions['region'].values) == {region}
+        assert set(predictions['layer'].values) == {expected_layer}
+
+
+class TestLayerMappedModel:
+    @pytest.mark.parametrize(['model_ctr', 'layers', 'region'], [
+        (pytorch_custom, 'relu2', 'IT'),
+        (pytorch_custom, ['linear', 'relu2'], 'IT'),
+    ])
+    def test_commit(self, model_ctr, layers, region):
+        activations_model = model_ctr()
+        layer_model = LayerMappedModel(identifier=activations_model.identifier, activations_model=activations_model,
+                                       region_layer_map={region: layers})
+
+        layer_model.start_recording(region)
+        stimulus_set = StimulusSet([{'stimulus_id': 'test'}])
+        stimulus_set.stimulus_paths = {'test': os.path.join(os.path.dirname(__file__), 'rgb1.jpg')}
+        stimulus_set.identifier = self.__class__.__name__
+        predictions = layer_model.look_at(stimulus_set)
+        assert set(predictions['region'].values) == {region}
+        assert set(predictions['layer'].values) == {layers} if isinstance(layers, str) else set(layers)