brainscore-vision 2.1__py3-none-any.whl

brainscore_vision/benchmarks/imagenet/test.py

@@ -0,0 +1,32 @@
+import numpy as np
+from pytest import approx
+
+from brainio.assemblies import BehavioralAssembly
+from brainscore_vision.benchmarks.imagenet import Imagenet2012
+from brainscore_vision.model_interface import BrainModel
+
+
+class TestImagenet2012:
+    def test_groundtruth(self):
+        benchmark = Imagenet2012()
+        source = benchmark._stimulus_set
+
+        class GroundTruth(BrainModel):
+            def start_task(self, task, fitting_stimuli):
+                assert task == BrainModel.Task.label
+                assert fitting_stimuli == 'imagenet'  # shortcut
+
+            def look_at(self, stimuli, number_of_trials=1):
+                source_image_ids = source['stimulus_id'].values
+                stimuli_image_ids = stimuli['stimulus_id'].values
+                sorted_x = source_image_ids[np.argsort(source_image_ids)]
+                sorted_index = np.searchsorted(sorted_x, stimuli_image_ids)
+                aligned_source = source.loc[sorted_index]
+                labels = aligned_source['synset'].values
+                return BehavioralAssembly([labels], coords={
+                    **{column: ('presentation', aligned_source[column].values) for column in aligned_source.columns},
+                    **{'choice': ('choice', ['dummy'])}}, dims=['choice', 'presentation'])
+
+        candidate = GroundTruth()
+        score = benchmark(candidate)
+        assert score == approx(1)

brainscore_vision/benchmarks/imagenet_c/__init__.py

@@ -0,0 +1,7 @@
+from brainscore_vision import benchmark_registry
+from .benchmark import Imagenet_C_Noise, Imagenet_C_Blur, Imagenet_C_Weather, Imagenet_C_Digital
+
+benchmark_registry['ImageNet-C-noise-top1'] = Imagenet_C_Noise
+benchmark_registry['ImageNet-C-blur-top1'] = Imagenet_C_Blur
+benchmark_registry['ImageNet-C-weather-top1'] = Imagenet_C_Weather
+benchmark_registry['ImageNet-C-digital-top1'] = Imagenet_C_Digital

brainscore_vision/benchmarks/imagenet_c/benchmark.py

@@ -0,0 +1,204 @@
+import logging
+import os
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+import xarray as xr
+
+from brainio.fetch import StimulusSetLoader
+from brainio.stimuli import StimulusSet
+from brainscore_core import Score
+from brainscore_vision import load_stimulus_set, load_metric
+from brainscore_vision.benchmarks import BenchmarkBase
+from brainscore_vision.benchmarks.imagenet.benchmark import NUMBER_OF_TRIALS
+from brainscore_vision.model_interface import BrainModel
+
+_logger = logging.getLogger(__name__)
+LOCAL_STIMULUS_DIRECTORY = '/braintree/data2/active/common/imagenet-c-brainscore-stimuli/'
+
+BIBTEX = """@ARTICLE{Hendrycks2019-di,
+   title         = "Benchmarking Neural Network Robustness to Common Corruptions
+                    and Perturbations",
+   author        = "Hendrycks, Dan and Dietterich, Thomas",
+   abstract      = "In this paper we establish rigorous benchmarks for image
+                    classifier robustness. Our first benchmark, ImageNet-C,
+                    standardizes and expands the corruption robustness topic,
+                    while showing which classifiers are preferable in
+                    safety-critical applications. Then we propose a new dataset
+                    called ImageNet-P which enables researchers to benchmark a
+                    classifier's robustness to common perturbations. Unlike
+                    recent robustness research, this benchmark evaluates
+                    performance on common corruptions and perturbations not
+                    worst-case adversarial perturbations. We find that there are
+                    negligible changes in relative corruption robustness from
+                    AlexNet classifiers to ResNet classifiers. Afterward we
+                    discover ways to enhance corruption and perturbation
+                    robustness. We even find that a bypassed adversarial defense
+                    provides substantial common perturbation robustness.
+                    Together our benchmarks may aid future work toward networks
+                    that robustly generalize.",
+   month         =  mar,
+   year          =  2019,
+   archivePrefix = "arXiv",
+   primaryClass  = "cs.LG",
+   eprint        = "1903.12261",
+   url           = "https://arxiv.org/abs/1903.12261"
+}"""
+
+def Imagenet_C_Noise(sampling_factor=10):
+    return Imagenet_C_Category('noise', sampling_factor=sampling_factor)
+
+def Imagenet_C_Blur(sampling_factor=10):
+    return Imagenet_C_Category('blur', sampling_factor=sampling_factor)
+
+def Imagenet_C_Weather(sampling_factor=10):
+    return Imagenet_C_Category('weather', sampling_factor=sampling_factor)
+
+def Imagenet_C_Digital(sampling_factor=10):
+    return Imagenet_C_Category('digital', sampling_factor=sampling_factor)
+
+
+class Imagenet_C_Category(BenchmarkBase):
+    """
+    Runs all ImageNet C benchmarks within a noise category, ie: 
+    gaussian noise [1-5]
+    shot noise [1-5]
+    impulse noise [1-5]
+    """
+    noise_category_map = {
+        'noise'   : ['gaussian_noise', 'shot_noise', 'impulse_noise'],
+        'blur'    : ['glass_blur', 'motion_blur', 'zoom_blur', 'defocus_blur'],
+        'weather' : ['snow', 'frost', 'fog', 'brightness'],
+        'digital' : ['pixelate', 'contrast', 'elastic_transform', 'jpeg_compression']
+    }
+
+    def __init__(self, noise_category, sampling_factor=10):
+        self.noise_category = noise_category
+        self.stimulus_set_name = f'imagenet_c.{noise_category}'
+
+        self.sampling_factor = sampling_factor
+        self.stimulus_set = self.load_stimulus_set()
+        self.noise_types = self.noise_category_map[noise_category]
+
+        ceiling = Score(1)
+        super(Imagenet_C_Category, self).__init__(identifier=f'ImageNet-C-{noise_category}-top1',
+                                                  version=2,
+                                                  ceiling_func=lambda: ceiling,
+                                                  parent='Hendrycks2019-top1',
+                                                  bibtex=BIBTEX)
+
+
+    def load_stimulus_set(self):
+        """
+        ImageNet-C is quite large, and thus cumbersome to download each time the benchmark is run.
+        Here we try loading a local copy first, before proceeding to download the AWS copy.
+        """
+        try:
+            _logger.debug(f'Loading local Imagenet-C {self.noise_category}')
+            category_path = os.path.join(
+                LOCAL_STIMULUS_DIRECTORY,
+                f'stimulus_imagenet_c_{self.noise_category}'
+            )
+            loader = SampledStimulusSetLoader(
+                cls=StimulusSet,
+                csv_path=os.path.join(category_path, f'stimulus_imagenet_c_{self.noise_category}.csv'),
+                stimuli_directory=category_path,
+                sampling_factor=self.sampling_factor
+            )
+
+            return loader.load()
+
+        except OSError as error:
+            _logger.debug(f'Excepted {error}. Attempting to access {self.stimulus_set_name} through Brainscore.')
+            return load_stimulus_set(self.stimulus_set_name)
+
+    def __call__(self, candidate):
+        scores = xr.concat([
+            Imagenet_C_Type(self.stimulus_set, noise_type, self.noise_category)(candidate)
+            for noise_type in self.noise_types
+        ], dim='presentation')
+        assert len(set(scores['noise_type'].values)) == len(self.noise_types)
+        center = np.mean(scores)
+        error = np.std(scores)
+        score = Score(center)
+        score.attrs['error'] = error
+        score.attrs[Score.RAW_VALUES_KEY] = scores
+        return score
+
+
+class Imagenet_C_Type(BenchmarkBase):
+    """
+    Runs a group in imnet C benchmarks, like gaussian noise [1-5]
+    """
+    def __init__(self, stimulus_set, noise_type, noise_category):
+        self.stimulus_set = stimulus_set[stimulus_set['noise_type'] == noise_type]
+        self.noise_type = noise_type
+        self.noise_category = noise_category
+        ceiling = Score(1)
+        super(Imagenet_C_Type, self).__init__(identifier=f'Hendrycks2019-{noise_category}-{noise_type}-top1',
+                                              version=2,
+                                              ceiling_func=lambda: ceiling,
+                                              parent=f'Hendrycks2019-{noise_category}-top1',
+                                              bibtex=BIBTEX)
+
+    def __call__(self, candidate):
+        score = xr.concat([
+            Imagenet_C_Individual(self.stimulus_set, noise_level, self.noise_type, self.noise_category)(candidate)
+            for noise_level in range(1, 6)
+        ], dim='presentation')
+        return score
+
+
+class Imagenet_C_Individual(BenchmarkBase):
+    """
+    Runs an individual ImageNet C benchmark, like "gaussian_noise_1"
+    """
+
+    def __init__(self, stimulus_set, noise_level, noise_type, noise_category):
+        self.stimulus_set = stimulus_set[stimulus_set['noise_level'] == noise_level]
+        self.noise_level = noise_level
+        self.noise_type = noise_type
+        self.benchmark_name = f'Hendrycks2019-{noise_category}-{noise_type}-{noise_level}-top1'
+        self._similarity_metric = load_metric('accuracy')
+        ceiling = Score(1)
+        super(Imagenet_C_Individual, self).__init__(identifier=self.benchmark_name, version=2,
+                                                    ceiling_func=lambda: ceiling,
+                                                    parent=f'Hendrycks2019-{noise_category}-{noise_type}-top1',
+                                                    bibtex=BIBTEX)
+
+    def __call__(self, candidate):
+        candidate.start_task(BrainModel.Task.label, 'imagenet')
+        stimulus_set = self.stimulus_set[
+            list(set(self.stimulus_set.columns) - {'synset'})].copy().reset_index()  # do not show label
+        stimulus_set.identifier = f'{self.benchmark_name}-{len(stimulus_set)}samples'
+        predictions = candidate.look_at(stimulus_set, number_of_trials=NUMBER_OF_TRIALS)
+        score = self._similarity_metric(
+            predictions.sortby('filename'),
+            self.stimulus_set.sort_values('filename')['synset'].values
+        ).raw
+
+        score = score.assign_coords(
+            name=('presentation', [f'{self.benchmark_name}' for _ in range(len(score.presentation))])
+        )
+
+        return score
+
+
+class SampledStimulusSetLoader(StimulusSetLoader):
+    """
+    Subclass of StimulusSetLoader that allows for downsampling of the stimulus set before loading.
+    """
+    def __init__(self, cls, csv_path, stimuli_directory, sampling_factor):
+        super().__init__(cls, csv_path, stimuli_directory)
+        self.sampling_factor = sampling_factor
+
+    def load(self):
+        stimulus_set = pd.read_csv(self.csv_path)[::self.sampling_factor]
+        self.correct_stimulus_id_name(stimulus_set)
+        stimulus_set = self.stimulus_set_class(stimulus_set)
+        stimulus_set.stimulus_paths = {row['stimulus_id']: Path(self.stimuli_directory) / row['filename']
+                                       for _, row in stimulus_set.iterrows()}
+        # make sure that all the stimulus files a loaded StimulusSet offers access to are actually available
+        assert all(stimulus_path.is_file() for stimulus_path in stimulus_set.stimulus_paths.values())
+        return stimulus_set

brainscore_vision/benchmarks/imagenet_c/test.py

@@ -0,0 +1,57 @@
+import numpy as np
+import pytest
+from pytest import approx
+
+from brainio.assemblies import BehavioralAssembly
+from brainscore_vision.benchmarks.imagenet_c import Imagenet_C_Noise, Imagenet_C_Blur, \
+    Imagenet_C_Weather, Imagenet_C_Digital
+from brainscore_vision.model_interface import BrainModel
+
+# downloads all ImageNet C benchmarks (50.3G) and runs with default downsampling by a factor of 10
+@pytest.mark.slow
+class TestImagenetC:
+    def test_groundtruth(self):
+        benchmarks = [
+            Imagenet_C_Noise(),
+            Imagenet_C_Blur(),
+            Imagenet_C_Weather(),
+            Imagenet_C_Digital(),
+        ]
+
+        class Static(BrainModel):
+            def start_task(self, task, fitting_stimuli):
+                assert task == BrainModel.Task.label
+                assert fitting_stimuli == 'imagenet'  # shortcut
+
+            def look_at(self, stimuli, number_of_trials=1):
+                labels = -np.ones_like(stimuli['stimulus_id'].values)
+                return BehavioralAssembly([labels], coords={
+                    **{column: ('presentation', stimuli[column].values) for column in stimuli.columns},
+                    **{'choice': ('choice', ['dummy'])}}, dims=['choice', 'presentation'])
+
+        candidate = Static()
+        scores = [benchmark(candidate) for benchmark in benchmarks]
+        assert all([np.mean(score) == approx(0) for score in scores])
+
+# downloads ImageNet C blur benchmarks (7.1G) and downsamples with a factor of 1000
+@pytest.mark.travis_slow
+class TestImagenetC_Category:
+    def test_groundtruth(self):
+        benchmarks = [
+            Imagenet_C_Blur(sampling_factor=1000),
+        ]
+
+        class Static(BrainModel):
+            def start_task(self, task, fitting_stimuli):
+                assert task == BrainModel.Task.label
+                assert fitting_stimuli == 'imagenet'  # shortcut
+
+            def look_at(self, stimuli, number_of_trials=1):
+                labels = -np.ones_like(stimuli['stimulus_id'].values)
+                return BehavioralAssembly([labels], coords={
+                    **{column: ('presentation', stimuli[column].values) for column in stimuli.columns},
+                    **{'choice': ('choice', ['dummy'])}}, dims=['choice', 'presentation'])
+
+        candidate = Static()
+        scores = [benchmark(candidate) for benchmark in benchmarks]
+        assert all([np.mean(score) == approx(0) for score in scores])

brainscore_vision/benchmarks/islam2021/__init__.py

@@ -0,0 +1,11 @@
+from brainscore_vision import benchmark_registry
+from .benchmark import _Islam2021Dimensionality
+
+benchmark_registry['Islam2021-shape_v1_dimensionality'] = lambda: _Islam2021Dimensionality("V1", "shape")
+benchmark_registry['Islam2021-texture_v1_dimensionality'] = lambda: _Islam2021Dimensionality("V1", "texture")
+benchmark_registry['Islam2021-shape_v2_dimensionality'] = lambda: _Islam2021Dimensionality("V2", "shape")
+benchmark_registry['Islam2021-texture_v2_dimensionality'] = lambda: _Islam2021Dimensionality("V2", "texture")
+benchmark_registry['Islam2021-shape_v4_dimensionality'] = lambda: _Islam2021Dimensionality("V4", "shape")
+benchmark_registry['Islam2021-texture_v4_dimensionality'] = lambda: _Islam2021Dimensionality("V4", "texture")
+benchmark_registry['Islam2021-shape_it_dimensionality'] = lambda: _Islam2021Dimensionality("IT", "shape")
+benchmark_registry['Islam2021-texture_it_dimensionality'] = lambda: _Islam2021Dimensionality("IT", "texture")

brainscore_vision/benchmarks/islam2021/benchmark.py

@@ -0,0 +1,107 @@
+import random
+
+import numpy as np
+from tqdm import tqdm
+
+from brainscore_core import Score
+from brainscore_core.benchmarks import BenchmarkBase
+from brainscore_vision import BrainModel, load_stimulus_set, load_metric
+from brainscore_vision.metrics.dimensionality import Dimensionality
+
+BIBTEX = """@inproceedings{
+            islam2021shape,
+            title={Shape or Texture: Understanding Discriminative Features in {\{}CNN{\}}s},
+            author={Md Amirul Islam and Matthew Kowal and Patrick Esser and Sen Jia and Bj{\"o}rn Ommer and Konstantinos G. Derpanis and Neil Bruce},
+            booktitle={International Conference on Learning Representations},
+            year={2021},
+            url={https://openreview.net/forum?id=NcFEZOi-rLa}
+        }"""
+
+
+TIME_BIN_ST, TIME_BIN_END = 70, 170  # standard core object recognition response, following Majaj*, Hong*, et al. 2015
+SEED = 1751 #turn this benchmark into a deterministic one
+
+class _Islam2021Dimensionality(BenchmarkBase):
+    def __init__(self,region,factor,deterministic=True):
+        assert factor in ["shape","texture","residual"]
+        factor_idx = {"shape":"shape", "texture":"texture", "residual":"residual"}[factor]
+        assert region in ["V1","V2","V4","IT"]
+        self.stimulus_set  = load_stimulus_set("Islam2021")
+        self.region = region
+        self.deterministic = deterministic
+        self._metric = load_metric('factor_dimensionality', factor=factor_idx)
+        self._number_of_trials = 1
+        super(_Islam2021Dimensionality, self).__init__(
+            identifier=f'Islam2021-{region + "_" + factor + "_dimensionality"}', version=1,
+            ceiling=Score(1),
+            parent='Islam2021',
+            bibtex=BIBTEX)
+
+    def set_generator(self):
+        if self.deterministic:
+            self.generator = random.Random(SEED)
+        else:
+            self.generator = random.Random()
+
+    def __call__(self, candidate: BrainModel):
+        self.set_generator()
+        candidate.start_recording(self.region,[(TIME_BIN_ST, TIME_BIN_END)])
+        assembly = candidate.look_at(self.stimulus_set)
+        factors, assembly1, assembly2 = self.get_assembly_sets(assembly)
+        assembly1 = self.prepare_assembly(assembly1,factors)
+        assembly2 = self.prepare_assembly(assembly2,factors)
+        score = self._metric(assembly1, assembly2)
+        return score
+
+    def get_assembly_sets(self,assembly,samples=None):
+        textures = list(set(self.stimulus_set["texture"].values))
+        textures.sort()
+        assert len(textures) == 5
+        shapes = list(set(self.stimulus_set["shape"].values))
+        shapes.sort()
+        assert len(shapes) == 20
+        factors, indexes1, indexes2 = [], [], []
+        if samples is None:
+            samples = len(self.stimulus_set)
+        for idx1 in tqdm(range(samples)):
+            factor, idx2 = self.get_index_pair(idx1,textures,shapes)
+            indexes1.append(idx1)
+            indexes2.append(idx2)
+            factors.append(factor)
+        return np.array(factors), assembly[:,np.array(indexes1)], assembly[:,np.array(indexes2)]
+
+    def get_index_pair(self,idx1,textures,shapes):
+        sample1 = self.stimulus_set.iloc[idx1]
+        factor = self.generator.choice(["shape", "texture"])
+        if factor == "shape": # same shape, different texture
+            list_possible_textures = textures.copy()
+            list_possible_textures.remove(sample1["texture"])
+            texture2 = self.generator.choice(list_possible_textures)
+            possible_samples_cond = \
+                (self.stimulus_set["original_image_id"] == sample1["original_image_id"]) & \
+                (self.stimulus_set["texture"] == texture2)
+            idx2 = np.where(possible_samples_cond)[0].item()
+            sample2 = self.stimulus_set.iloc[idx2]
+            assert sample2["shape"] == sample1["shape"] and sample1["texture"] != sample2["texture"]
+
+        else: #different shape, same texture
+            list_possible_shapes = shapes.copy()
+            list_possible_shapes.remove(sample1["shape"])
+            shape2 = self.generator.choice(list_possible_shapes)
+            possible_samples_cond = \
+                (self.stimulus_set["texture"] == sample1["texture"]) & \
+                (self.stimulus_set["shape"] == shape2)
+            possible_indexes = np.where(possible_samples_cond)[0]
+            idx2 = self.generator.choice(possible_indexes)
+            sample2 = self.stimulus_set.iloc[idx2]
+            assert sample2["shape"] != sample1["shape"] and sample1["texture"] == sample2["texture"]
+        return factor, idx2
+
+    def prepare_assembly(self,assembly,factors):
+        # prepare data assembly for the dimensionality metric
+        assert assembly.shape[1] == len(factors)
+        dim = assembly.dims[1]
+        assembly = assembly.assign_coords(factor = (dim,factors))
+        assembly = assembly.T
+        return assembly
+

brainscore_vision/benchmarks/islam2021/test.py

@@ -0,0 +1,47 @@
+from pathlib import Path
+
+import boto3
+import pytest
+from pytest import approx
+
+from brainio.assemblies import DataAssembly
+from brainscore_vision import load_benchmark
+from brainscore_vision.benchmark_helpers import PrecomputedFeatures
+
+
+@pytest.fixture()
+def alexnet_features():
+    # load
+    filename = 'alexnet-islam2021-classifier.5.nc'
+    precomputed_features_path = Path(__file__).parent / filename
+    if not precomputed_features_path.is_file():  # download on demand
+        s3 = boto3.client('s3')
+        s3.download_file('brain-score-tests', f'tests/test_benchmarks/{filename}',
+                         str(precomputed_features_path.absolute()))
+    precomputed_features = DataAssembly.from_files(file_path=precomputed_features_path)
+
+    # adjust metadata
+    stimulus_id = list(map(lambda x: x.split("/")[-1][:-4], precomputed_features['stimulus_path'].values))
+    precomputed_features['stimulus_path'] = stimulus_id
+    precomputed_features = precomputed_features.rename({'stimulus_path': 'stimulus_id'})
+    precomputed_features = precomputed_features.stack(presentation=('stimulus_id',))
+    precomputed_features = PrecomputedFeatures(precomputed_features, visual_degrees=None)
+    return precomputed_features
+
+
+@pytest.mark.private_access
+class TestEngineering:
+    @pytest.mark.parametrize('factor, area, expected_value', [
+        ('shape', 'v1', approx(0.18310547, abs=0.001)),
+        ('texture', 'v1', approx(0.30834961, abs=0.001)),
+        ('shape', 'v2', approx(0.18310547, abs=0.001)),
+        ('texture', 'v2', approx(0.30834961, abs=0.001)),
+        ('shape', 'v4', approx(0.18310547, abs=0.001)),
+        ('texture', 'v4', approx(0.30834961, abs=0.001)),
+        ('shape', 'it', approx(0.18310547, abs=0.001)),
+        ('texture', 'it', approx(0.30834961, abs=0.001)),
+    ])
+    def test_dimensionality(self, factor, area, expected_value, alexnet_features):
+        benchmark = load_benchmark(f'Islam2021-{factor}_{area}_dimensionality')
+        score = benchmark(alexnet_features)
+        assert score.item() == expected_value

brainscore_vision/benchmarks/kar2019/__init__.py

@@ -0,0 +1,4 @@
+from brainscore_vision import benchmark_registry
+from .benchmark import DicarloKar2019OST
+
+benchmark_registry['Kar2019-ost'] = DicarloKar2019OST

brainscore_vision/benchmarks/kar2019/benchmark.py

@@ -0,0 +1,88 @@
+import numpy as np
+
+from brainscore_core import Score
+from brainscore_vision import load_dataset, load_metric
+from brainscore_vision.benchmark_helpers.screen import place_on_screen
+from brainscore_vision.benchmarks import BenchmarkBase, ceil_score
+from brainscore_vision.model_interface import BrainModel
+
+BIBTEX = """@Article{Kar2019,
+            author={Kar, Kohitij
+            and Kubilius, Jonas
+            and Schmidt, Kailyn
+            and Issa, Elias B.
+            and DiCarlo, James J.},
+            title={Evidence that recurrent circuits are critical to the ventral stream's execution of core object recognition behavior},
+            journal={Nature Neuroscience},
+            year={2019},
+            month={Jun},
+            day={01},
+            volume={22},
+            number={6},
+            pages={974-983},
+            abstract={Non-recurrent deep convolutional neural networks (CNNs) are currently the best at modeling core object recognition, a behavior that is supported by the densely recurrent primate ventral stream, culminating in the inferior temporal (IT) cortex. If recurrence is critical to this behavior, then primates should outperform feedforward-only deep CNNs for images that require additional recurrent processing beyond the feedforward IT response. Here we first used behavioral methods to discover hundreds of these `challenge' images. Second, using large-scale electrophysiology, we observed that behaviorally sufficient object identity solutions emerged {\textasciitilde}30{\thinspace}ms later in the IT cortex for challenge images compared with primate performance-matched `control' images. Third, these behaviorally critical late-phase IT response patterns were poorly predicted by feedforward deep CNN activations. Notably, very-deep CNNs and shallower recurrent CNNs better predicted these late IT responses, suggesting that there is a functional equivalence between additional nonlinear transformations and recurrence. Beyond arguing that recurrent circuits are critical for rapid object identification, our results provide strong constraints for future recurrent model development.},
+            issn={1546-1726},
+            doi={10.1038/s41593-019-0392-5},
+            url={https://doi.org/10.1038/s41593-019-0392-5}
+            }"""
+VISUAL_DEGREES = 8
+NUMBER_OF_TRIALS = 44
+TIME_BINS = [(time_bin_start, time_bin_start + 10) for time_bin_start in range(70, 250, 10)]
+
+
+class DicarloKar2019OST(BenchmarkBase):
+    def __init__(self):
+        ceiling = Score(.79)  # computed offline by Kohitij Kar
+        super(DicarloKar2019OST, self).__init__(identifier='Kar2019-ost', version=2,
+                                                ceiling_func=lambda: ceiling,
+                                                parent='IT',
+                                                bibtex=BIBTEX)
+        assembly = load_dataset('Kar2019')
+        # drop duplicate images
+        _, index = np.unique(assembly['stimulus_id'], return_index=True)
+        assembly = assembly.isel(presentation=index)
+        assembly.attrs['stimulus_set'] = assembly.stimulus_set.drop_duplicates('stimulus_id')
+
+        assembly = assembly.sel(decoder='svm')
+
+        self._assembly = assembly
+        self._assembly['truth'] = self._assembly['image_label']
+        self._assembly.stimulus_set['truth'] = self._assembly.stimulus_set['image_label']
+
+        self._similarity_metric = load_metric('ost')
+
+        self._visual_degrees = VISUAL_DEGREES
+        self._number_of_trials = NUMBER_OF_TRIALS
+        self._time_bins = TIME_BINS
+
+    def __call__(self, candidate: BrainModel) -> Score:
+        candidate.start_recording('IT', time_bins=self._time_bins)
+        stimulus_set = place_on_screen(self._assembly.stimulus_set, target_visual_degrees=candidate.visual_degrees(),
+                                       source_visual_degrees=self._visual_degrees)
+        # Temporal recordings from large candidates take up a lot of memory and compute time.
+        # In order to quickly reject recordings that are static over time,
+        # we will show one image and check whether the recordings vary over time at all or not.
+        # If they don't we can quickly score the candidate with a failure state
+        # since it will not be able to predict temporal differences with the OST metric
+        check_stimulus_set = stimulus_set[:1]
+        check_stimulus_set.identifier = None  # unset identifier to avoid storing (interferes with actual stimulus_set)
+        check_recordings = candidate.look_at(check_stimulus_set, number_of_trials=self._number_of_trials)
+        if not temporally_varying(check_recordings):
+            score = Score(np.nan)
+        else:
+            recordings = candidate.look_at(stimulus_set, number_of_trials=self._number_of_trials)
+            score = self._similarity_metric(recordings, self._assembly)
+        score = ceil_score(score, self.ceiling)
+        return score
+
+
+def temporally_varying(recordings):
+    """
+    Tests whether the given recordings change over time, for any of the stimuli on any of the neuroids
+
+    :return True if any of the neuroids changes over time for any of the stimuli, False otherwise
+    """
+    recordings = recordings.transpose('presentation', 'neuroid', 'time_bin')
+    first_response = recordings.sel(time_bin=recordings['time_bin'].values[0])
+    different = recordings != first_response
+    return different.any()

brainscore_vision/benchmarks/kar2019/test.py

@@ -0,0 +1,93 @@
+from pathlib import Path
+
+import numpy as np
+import pytest
+from numpy.random.mtrand import RandomState
+from pytest import approx
+
+from brainio.assemblies import NeuroidAssembly, DataAssembly
+from brainscore_vision import load_benchmark
+from brainscore_vision.benchmark_helpers import PrecomputedFeatures
+from brainscore_vision.benchmark_helpers.test_helper import VisualDegreesTests, NumberOfTrialsTests
+from brainscore_vision.benchmarks.kar2019 import DicarloKar2019OST
+from brainscore_vision.data_helpers import s3
+
+visual_degrees = VisualDegreesTests()
+number_trials = NumberOfTrialsTests()
+
+
+@pytest.mark.memory_intense
+@pytest.mark.private_access
+@pytest.mark.slow
+def test_Kar2019ost_cornet_s():
+    benchmark = load_benchmark('Kar2019-ost')
+    filename = 'cornet_s-kar2019.nc'
+    filepath = Path(__file__).parent / filename
+    s3.download_file_if_not_exists(local_path=filepath,
+                                   bucket='brainscore-unittests', remote_filepath=f'tests/test_benchmarks/{filename}')
+    precomputed_features = NeuroidAssembly.from_files(
+        filepath,
+        stimulus_set_identifier=benchmark._assembly.stimulus_set.identifier,
+        stimulus_set=benchmark._assembly.stimulus_set)
+    precomputed_features = PrecomputedFeatures(precomputed_features, visual_degrees=8)
+    # score
+    score = benchmark(precomputed_features).raw
+    assert score == approx(.316, abs=.005)
+
+
+@pytest.mark.private_access
+@pytest.mark.parametrize('benchmark, candidate_degrees, image_id, expected', [
+    pytest.param('Kar2019-ost', 14, '6d19b24c29832dfb28360e7731e3261c13a4287f',
+                 approx(.225021, abs=.0001), marks=[pytest.mark.private_access]),
+    pytest.param('Kar2019-ost', 6, '6d19b24c29832dfb28360e7731e3261c13a4287f',
+                 approx(.001248, abs=.0001), marks=[pytest.mark.private_access]),
+])
+def test_amount_gray(benchmark: str, candidate_degrees: int, image_id: str, expected: float):
+    visual_degrees.amount_gray_test(benchmark, candidate_degrees, image_id, expected)
+
+
+@pytest.mark.private_access
+def test_repetitions():
+    number_trials.repetitions_test('Kar2019-ost')
+
+
+@pytest.mark.memory_intense
+@pytest.mark.private_access
+def test_no_time():
+    benchmark = DicarloKar2019OST()
+    rnd = RandomState(0)
+    stimuli = benchmark._assembly.stimulus_set
+    source = DataAssembly(rnd.rand(len(stimuli), 5, 1), coords={
+        'stimulus_id': ('presentation', stimuli['stimulus_id']),
+        'image_label': ('presentation', stimuli['image_label']),
+        'truth': ('presentation', stimuli['truth']),
+        'neuroid_id': ('neuroid', list(range(5))),
+        'layer': ('neuroid', ['test'] * 5),
+        'time_bin_start': ('time_bin', [70]),
+        'time_bin_end': ('time_bin', [170]),
+    }, dims=['presentation', 'neuroid', 'time_bin'])
+    source.name = __name__ + ".test_notime"
+    score = benchmark(PrecomputedFeatures(source, visual_degrees=8))
+    assert np.isnan(score)  # not a temporal model
+    assert np.isnan(score.raw)  # not a temporal model
+    assert score.attrs['ceiling'] == approx(.79)
+
+
+@pytest.mark.memory_intense
+@pytest.mark.private_access
+def test_random_time():
+    benchmark = DicarloKar2019OST()
+    rnd = RandomState(0)
+    stimuli = benchmark._assembly.stimulus_set
+    source = DataAssembly(rnd.rand(len(stimuli), 5, 5), coords={
+        'stimulus_id': ('presentation', stimuli['stimulus_id']),
+        'image_label': ('presentation', stimuli['image_label']),
+        'truth': ('presentation', stimuli['truth']),
+        'neuroid_id': ('neuroid', list(range(5))),
+        'layer': ('neuroid', ['test'] * 5),
+        'time_bin_start': ('time_bin', [70, 90, 110, 130, 150]),
+        'time_bin_end': ('time_bin', [90, 110, 130, 150, 170]),
+    }, dims=['presentation', 'neuroid', 'time_bin'])
+    source.name = __name__ + ".test_notime"
+    score = benchmark(PrecomputedFeatures(source, visual_degrees=8))
+    assert np.isnan(score)  # not a good temporal model