brainscore-vision 2.1__py3-none-any.whl

brainscore_vision/benchmarks/rajalingham2018/benchmarks/benchmark.py

@@ -0,0 +1,74 @@
+import numpy as np
+
+from brainscore_core import Score
+from brainscore_vision import load_metric, load_stimulus_set, load_dataset
+from brainscore_vision.benchmark_helpers.screen import place_on_screen
+from brainscore_vision.benchmarks import BenchmarkBase
+from brainscore_vision.metric_helpers.transformations import apply_aggregate
+from brainscore_vision.model_interface import BrainModel
+from brainscore_vision.utils import LazyLoad
+
+BIBTEX = """@article {Rajalingham240614,
+                author = {Rajalingham, Rishi and Issa, Elias B. and Bashivan, Pouya and Kar, Kohitij and Schmidt, Kailyn and DiCarlo, James J.},
+                title = {Large-scale, high-resolution comparison of the core visual object recognition behavior of humans, monkeys, and state-of-the-art deep artificial neural networks},
+                elocation-id = {240614},
+                year = {2018},
+                doi = {10.1101/240614},
+                publisher = {Cold Spring Harbor Laboratory},
+                abstract = {Primates{\textemdash}including humans{\textemdash}can typically recognize objects in visual images at a glance even in the face of naturally occurring identity-preserving image transformations (e.g. changes in viewpoint). A primary neuroscience goal is to uncover neuron-level mechanistic models that quantitatively explain this behavior by predicting primate performance for each and every image. Here, we applied this stringent behavioral prediction test to the leading mechanistic models of primate vision (specifically, deep, convolutional, artificial neural networks; ANNs) by directly comparing their behavioral signatures against those of humans and rhesus macaque monkeys. Using high-throughput data collection systems for human and monkey psychophysics, we collected over one million behavioral trials for 2400 images over 276 binary object discrimination tasks. Consistent with previous work, we observed that state-of-the-art deep, feed-forward convolutional ANNs trained for visual categorization (termed DCNNIC models) accurately predicted primate patterns of object-level confusion. However, when we examined behavioral performance for individual images within each object discrimination task, we found that all tested DCNNIC models were significantly non-predictive of primate performance, and that this prediction failure was not accounted for by simple image attributes, nor rescued by simple model modifications. These results show that current DCNNIC models cannot account for the image-level behavioral patterns of primates, and that new ANN models are needed to more precisely capture the neural mechanisms underlying primate object vision. To this end, large-scale, high-resolution primate behavioral benchmarks{\textemdash}such as those obtained here{\textemdash}could serve as direct guides for discovering such models.SIGNIFICANCE STATEMENT Recently, specific feed-forward deep convolutional artificial neural networks (ANNs) models have dramatically advanced our quantitative understanding of the neural mechanisms underlying primate core object recognition. In this work, we tested the limits of those ANNs by systematically comparing the behavioral responses of these models with the behavioral responses of humans and monkeys, at the resolution of individual images. Using these high-resolution metrics, we found that all tested ANN models significantly diverged from primate behavior. Going forward, these high-resolution, large-scale primate behavioral benchmarks could serve as direct guides for discovering better ANN models of the primate visual system.},
+                URL = {https://www.biorxiv.org/content/early/2018/02/12/240614},
+                eprint = {https://www.biorxiv.org/content/early/2018/02/12/240614.full.pdf},
+                journal = {bioRxiv}
+            }"""
+
+
+class _DicarloRajalingham2018(BenchmarkBase):
+    def __init__(self, metric, metric_identifier):
+        self._metric = metric
+        self._fitting_stimuli = load_stimulus_set('objectome.public')
+        self._assembly = LazyLoad(lambda: load_assembly('private'))
+        self._visual_degrees = 8
+        self._number_of_trials = 2
+        super(_DicarloRajalingham2018, self).__init__(
+            identifier='Rajalingham2018-' + metric_identifier, version=2,
+            ceiling_func=lambda: self._metric.ceiling(self._assembly),
+            parent='behavior',
+            bibtex=BIBTEX)
+
+    def __call__(self, candidate: BrainModel) -> Score:
+        fitting_stimuli = place_on_screen(self._fitting_stimuli, target_visual_degrees=candidate.visual_degrees(),
+                                          source_visual_degrees=self._visual_degrees)
+        candidate.start_task(BrainModel.Task.probabilities, fitting_stimuli)
+        stimulus_set = place_on_screen(self._assembly.stimulus_set, target_visual_degrees=candidate.visual_degrees(),
+                                       source_visual_degrees=self._visual_degrees)
+        probabilities = candidate.look_at(stimulus_set, number_of_trials=self._number_of_trials)
+        score = self._metric(probabilities, self._assembly)
+        ceiling = self.ceiling
+        score = self.ceil_score(score, ceiling)
+        return score
+
+    def ceil_score(self, score, ceiling):
+        assert set(score.raw['split'].values) == set(ceiling.raw['split'].values)
+        split_scores = []
+        for split in ceiling.raw['split'].values:
+            split_score = score.raw.sel(split=split)
+            split_ceiling = ceiling.raw.sel(split=split)
+            ceiled_split_score = split_score / np.sqrt(split_ceiling)
+            ceiled_split_score = ceiled_split_score.expand_dims('split')
+            ceiled_split_score['split'] = [split]
+            split_scores.append(ceiled_split_score)
+        split_scores = Score.merge(*split_scores)
+        split_scores = apply_aggregate(self._metric.aggregate, split_scores)
+        split_scores.attrs[Score.RAW_VALUES_KEY] = score  # this will override raw per-split ceiled scores which is ok
+        split_scores.attrs['ceiling'] = ceiling
+        return split_scores
+
+
+def DicarloRajalingham2018I2n():
+    return _DicarloRajalingham2018(metric=load_metric('i2n'), metric_identifier='i2n')
+
+
+def load_assembly(access='private'):
+    assembly = load_dataset(f'Rajalingham2018.{access}')
+    assembly['correct'] = assembly['choice'] == assembly['sample_obj']
+    return assembly

brainscore_vision/benchmarks/rajalingham2018/benchmarks/public_benchmark.py

@@ -0,0 +1,10 @@
+from brainscore_vision import load_metric
+from .benchmark import load_assembly as load_rajalingham2018, _DicarloRajalingham2018
+
+
+class RajalinghamMatchtosamplePublicBenchmark(_DicarloRajalingham2018):
+    def __init__(self):
+        super(RajalinghamMatchtosamplePublicBenchmark, self).__init__(
+            metric=load_metric('i2n'), metric_identifier='i2n')
+        self._assembly = load_rajalingham2018(access='public')
+        self._ceiling_func = lambda: self._metric.ceiling(self._assembly, skipna=True)

brainscore_vision/benchmarks/rajalingham2018/test.py

@@ -0,0 +1,125 @@
+import os
+from pathlib import Path
+
+import numpy as np
+import pytest
+import xarray as xr
+from pytest import approx
+
+from brainio.assemblies import BehavioralAssembly
+from brainscore_vision import benchmark_registry, load_benchmark, load_metric, load_model
+from brainscore_vision.benchmark_helpers import PrecomputedFeatures
+from brainscore_vision.benchmark_helpers.test_helper import VisualDegreesTests, NumberOfTrialsTests
+from brainscore_vision.benchmarks.rajalingham2018 import DicarloRajalingham2018I2n
+from brainscore_vision.benchmarks.rajalingham2018.benchmarks.benchmark import _DicarloRajalingham2018
+from brainscore_vision.data_helpers import s3
+from brainscore_vision.model_helpers.brain_transformation import ProbabilitiesMapping
+from brainscore_vision.model_interface import BrainModel
+
+visual_degrees = VisualDegreesTests()
+number_trials = NumberOfTrialsTests()
+
+
+@pytest.mark.parametrize('benchmark', [
+    'Rajalingham2018-i2n',
+    'Rajalingham2018public-i2n',
+])
+def test_benchmark_registry(benchmark):
+    assert benchmark in benchmark_registry
+
+
+@pytest.mark.private_access
+class TestRajalingham2018:
+    def test_ceiling(self):
+        benchmark = DicarloRajalingham2018I2n()
+        ceiling = benchmark.ceiling
+        assert ceiling == approx(.479, abs=.0064)
+
+    @pytest.mark.parametrize(['model', 'expected_score'],
+                             [
+                                 ('alexnet', .253),
+                                 ('resnet34', .37787),
+                                 ('resnet18', .3638),
+                             ])
+    def test_precomputed(self, model, expected_score):
+        benchmark = DicarloRajalingham2018I2n()
+        filepath = Path(__file__).parent / 'test_resources' / f'{model}-probabilities.nc'
+        stimulus_set = benchmark._assembly.stimulus_set
+        probabilities = BehavioralAssembly.from_files(
+            filepath, stimulus_set=stimulus_set, stimulus_set_identifier=stimulus_set.identifier)
+        candidate = PrecomputedProbabilities(probabilities)
+        score = benchmark(candidate)
+        assert score.raw == approx(expected_score, abs=.005)
+        assert score == approx(expected_score / np.sqrt(.479), abs=.005)
+
+
+class PrecomputedProbabilities(BrainModel):
+    def __init__(self, probabilities):
+        self.probabilities = probabilities
+
+    def visual_degrees(self) -> int:
+        return 8
+
+    def start_task(self, task: BrainModel.Task, fitting_stimuli):
+        assert task == BrainModel.Task.probabilities
+        assert len(fitting_stimuli) == 2160
+
+    def look_at(self, stimuli, number_of_trials=1):
+        assert set(self.probabilities['stimulus_id'].values) == set(stimuli['stimulus_id'].values)
+        image_ids = self.probabilities['stimulus_id'].values.tolist()
+        probabilities = self.probabilities[[image_ids.index(image_id) for image_id in stimuli['stimulus_id'].values], :]
+        assert all(probabilities['stimulus_id'].values == stimuli['stimulus_id'].values)
+        return probabilities
+
+
+def test_Rajalingham2018public():
+    benchmark = load_benchmark('Rajalingham2018public-i2n')
+    # load features
+    filename = 'CORnetZ-rajalingham2018public.nc'
+    filepath = Path(__file__).parent / filename
+    s3.download_file_if_not_exists(filepath,
+                                   bucket='brain-score-tests', remote_filepath=f'tests/test_benchmarks/{filename}')
+    precomputed_features = BehavioralAssembly.from_files(
+        filepath,
+        stimulus_set_identifier=benchmark._assembly.stimulus_set.identifier,
+        stimulus_set=benchmark._assembly.stimulus_set)
+    precomputed_features = PrecomputedFeatures(precomputed_features,
+                                               visual_degrees=8,  # doesn't matter, features are already computed
+                                               )
+    # score
+    score = benchmark(precomputed_features).raw
+    assert score == approx(.136923, abs=.005)
+
+
+@pytest.mark.parametrize('benchmark, candidate_degrees, image_id, expected', [
+    pytest.param('Rajalingham2018-i2n', 14, '0223bf9e5db0edad21976b16494fe9396a5ef145',
+                 approx(.225023, abs=.0001), marks=[pytest.mark.private_access]),
+    pytest.param('Rajalingham2018-i2n', 6, '0223bf9e5db0edad21976b16494fe9396a5ef145',
+                 approx(.002244, abs=.0001), marks=[pytest.mark.private_access]),
+    pytest.param('Rajalingham2018public-i2n', 14, '0020cef91bd626e9fbbabd853494ee444e5c9ecb',
+                 approx(.22486, abs=.0001), marks=[]),
+    pytest.param('Rajalingham2018public-i2n', 6, '0020cef91bd626e9fbbabd853494ee444e5c9ecb',
+                 approx(.00097, abs=.0001), marks=[]),
+])
+def test_amount_gray(benchmark: str, candidate_degrees: int, image_id: str, expected: float):
+    visual_degrees.amount_gray_test(benchmark, candidate_degrees, image_id, expected)
+
+
+@pytest.mark.private_access
+def test_repetitions():
+    number_trials.repetitions_test('Rajalingham2018-i2n')
+
+
+@pytest.mark.private_access
+class TestMetricScore:
+    @pytest.mark.parametrize(['model', 'expected_score'],
+                             [
+                                 ('alexnet', .253),
+                                 ('resnet50_tutorial', 0.348),
+                                 ('pixels', 0.0139)
+                             ])
+    def test_model(self, model, expected_score):
+        benchmark = load_benchmark('Rajalingham2018-i2n')
+        model = load_model(model)
+        score = benchmark(model)
+        assert score.raw == approx(expected_score, abs=0.005), f"expected {expected_score}, but got {score.raw}"

brainscore_vision/benchmarks/rajalingham2020/__init__.py

@@ -0,0 +1,4 @@
+from brainscore_vision import benchmark_registry
+from .benchmark import DicarloRajalingham2020ITPLS
+
+benchmark_registry['Rajalingham2020.IT-pls'] = DicarloRajalingham2020ITPLS

brainscore_vision/benchmarks/rajalingham2020/benchmark.py

@@ -0,0 +1,52 @@
+import numpy as np
+
+from brainscore_vision import load_metric, load_ceiling, load_dataset
+from brainscore_vision.benchmark_helpers.neural_common import NeuralBenchmark, average_repetition
+from brainscore_vision.utils import LazyLoad
+
+VISUAL_DEGREES = 8
+NUMBER_OF_TRIALS = 33
+BIBTEX = """@article {Rajalingham2020,
+            author = {Rajalingham, Rishi and Kar, Kohitij and Sanghavi, Sachi and Dehaene, Stanislas and DiCarlo, James J.},
+            title = {The inferior temporal cortex is a potential cortical precursor of orthographic processing in untrained monkeys},
+            journal = {Nature Communications},
+            year = {2020},
+            month = {Aug},
+            day = {04},
+            volume = {11},
+            number = {1},
+            pages = {3886},
+            abstract = {The ability to recognize written letter strings is foundational to human reading, but the underlying neuronal mechanisms remain largely unknown. Recent behavioral research in baboons suggests that non-human primates may provide an opportunity to investigate this question. We recorded the activity of hundreds of neurons in V4 and the inferior temporal cortex (IT) while na{\"i}ve macaque monkeys passively viewed images of letters, English words and non-word strings, and tested the capacity of those neuronal representations to support a battery of orthographic processing tasks. We found that simple linear read-outs of IT (but not V4) population responses achieved high performance on all tested tasks, even matching the performance and error patterns of baboons on word classification. These results show that the IT cortex of untrained primates can serve as a precursor of orthographic processing, suggesting that the acquisition of reading in humans relies on the recycling of a brain network evolved for other visual functions.},
+            issn = {2041-1723},
+            doi = {10.1038/s41467-020-17714-3},
+            url = {https://doi.org/10.1038/s41467-020-17714-3}}"""
+
+
+def _DicarloRajalingham2020Region(region, identifier_metric_suffix, similarity_metric, ceiler):
+    assembly_repetition = LazyLoad(lambda region=region: load_assembly(average_repetitions=False, region=region))
+    assembly = LazyLoad(lambda region=region: load_assembly(average_repetitions=True, region=region))
+    return NeuralBenchmark(identifier=f'Rajalingham2020.{region}-{identifier_metric_suffix}', version=1,
+                           assembly=assembly, similarity_metric=similarity_metric,
+                           visual_degrees=VISUAL_DEGREES, number_of_trials=NUMBER_OF_TRIALS,
+                           ceiling_func=lambda: ceiler(assembly_repetition),
+                           parent=region,
+                           bibtex=BIBTEX)
+
+
+def DicarloRajalingham2020ITPLS():
+    metric = load_metric('pls', crossvalidation_kwargs=dict(stratification_coord=None))
+    ceiler = load_ceiling('internal_consistency')
+    return _DicarloRajalingham2020Region('IT', identifier_metric_suffix='pls',
+                                         similarity_metric=metric, ceiler=ceiler)
+
+
+def load_assembly(average_repetitions, region):
+    assembly = load_dataset(identifier=f'Rajalingham2020')
+    assembly = assembly.sel(region=region)
+    assembly['region'] = 'neuroid', [region] * len(assembly['neuroid'])
+    assembly.load()
+    assembly = assembly.squeeze('time_bin')
+    assert NUMBER_OF_TRIALS == len(np.unique(assembly.coords['repetition']))
+    if average_repetitions:
+        assembly = average_repetition(assembly)
+    return assembly

brainscore_vision/benchmarks/rajalingham2020/test.py

@@ -0,0 +1,39 @@
+from pathlib import Path
+
+import pytest
+from pytest import approx
+
+from brainscore_vision.benchmark_helpers.test_helper import StandardizedTests, PrecomputedTests
+from brainscore_vision.data_helpers import s3
+
+standardized_tests = StandardizedTests()
+precomputed_test = PrecomputedTests()
+
+
+@pytest.mark.parametrize('benchmark, expected', [
+    pytest.param('Rajalingham2020.IT-pls', approx(.561013, abs=.001),
+                 marks=[pytest.mark.memory_intense, pytest.mark.slow]),
+])
+def test_ceilings(benchmark, expected):
+    standardized_tests.ceilings_test(benchmark, expected)
+
+
+@pytest.mark.parametrize('benchmark, visual_degrees, expected', [
+    pytest.param('Rajalingham2020.IT-pls', 8, approx(.693463, abs=.005),
+                 marks=[pytest.mark.memory_intense, pytest.mark.slow]),
+])
+def test_self_regression(benchmark, visual_degrees, expected):
+    standardized_tests.self_regression_test(benchmark, visual_degrees, expected)
+
+
+@pytest.mark.memory_intense
+@pytest.mark.slow
+@pytest.mark.parametrize('benchmark, expected', [
+    ('Rajalingham2020.IT-pls', approx(.147549, abs=.01)),
+])
+def test_Rajalingham2020(benchmark, expected):
+    filename = 'alexnet-rajalingham2020-features.12.nc'
+    filepath = Path(__file__).parent / filename
+    s3.download_file_if_not_exists(local_path=filepath,
+                                   bucket='brainscore-unittests', remote_filepath=f'tests/test_benchmarks/{filename}')
+    precomputed_test.run_test(benchmark=benchmark, precomputed_features_filepath=filepath, expected=expected)

brainscore_vision/benchmarks/sanghavi2020/__init__.py

@@ -0,0 +1,17 @@
+from brainscore_vision import benchmark_registry
+from .benchmarks.sanghavi2020_benchmark import DicarloSanghavi2020V4PLS, DicarloSanghavi2020ITPLS
+
+benchmark_registry['Sanghavi2020.V4-pls'] = DicarloSanghavi2020V4PLS
+benchmark_registry['Sanghavi2020.IT-pls'] = DicarloSanghavi2020ITPLS
+
+
+from .benchmarks.sanghavijozwik2020_benchmark import DicarloSanghaviJozwik2020V4PLS, DicarloSanghaviJozwik2020ITPLS
+
+benchmark_registry['SanghaviJozwik2020.V4-pls'] = DicarloSanghaviJozwik2020V4PLS
+benchmark_registry['SanghaviJozwik2020.IT-pls'] = DicarloSanghaviJozwik2020ITPLS
+
+
+from .benchmarks.sanghavimurty2020_benchmark import DicarloSanghaviMurty2020V4PLS, DicarloSanghaviMurty2020ITPLS
+
+benchmark_registry['SanghaviMurty2020.V4-pls'] = DicarloSanghaviMurty2020V4PLS
+benchmark_registry['SanghaviMurty2020.IT-pls'] = DicarloSanghaviMurty2020ITPLS

brainscore_vision/benchmarks/sanghavi2020/benchmarks/sanghavi2020_benchmark.py

@@ -0,0 +1,44 @@
+import numpy as np
+
+from brainscore_vision import load_ceiling, load_metric, load_dataset
+from brainscore_vision.benchmark_helpers.neural_common import NeuralBenchmark, average_repetition
+from brainscore_vision.data.sanghavi2020 import BIBTEX_SANGHAVI
+from brainscore_vision.utils import LazyLoad
+
+VISUAL_DEGREES = 8
+NUMBER_OF_TRIALS = 28
+
+
+def _DicarloSanghavi2020Region(region):
+    assembly_repetition = LazyLoad(lambda region=region: load_assembly(average_repetitions=False, region=region))
+    assembly = LazyLoad(lambda region=region: load_assembly(average_repetitions=True, region=region))
+    metric = load_metric('pls', crossvalidation_kwargs=dict(stratification_coord='object_name'))
+    ceiler = load_ceiling('internal_consistency')
+    return NeuralBenchmark(identifier=f'Sanghavi2020.{region}-pls', version=1,
+                           assembly=assembly, similarity_metric=metric,
+                           visual_degrees=VISUAL_DEGREES, number_of_trials=NUMBER_OF_TRIALS,
+                           ceiling_func=lambda: ceiler(assembly_repetition),
+                           parent=region,
+                           bibtex=BIBTEX_SANGHAVI)
+
+
+def DicarloSanghavi2020V4PLS():
+    return _DicarloSanghavi2020Region('V4')
+
+
+def DicarloSanghavi2020ITPLS():
+    return _DicarloSanghavi2020Region('IT')
+
+
+def load_assembly(average_repetitions, region):
+    assembly = load_dataset(f'Sanghavi2020')
+    assembly = assembly.sel(region=region)
+    assembly['region'] = 'neuroid', [region] * len(assembly['neuroid'])
+    assembly.load()
+    assembly = assembly.sel(time_bin_id=0)  # 70-170ms
+    assembly = assembly.squeeze('time_bin')
+    assert NUMBER_OF_TRIALS == len(np.unique(assembly.coords['repetition']))
+    assert VISUAL_DEGREES == assembly.attrs['image_size_degree']
+    if average_repetitions:
+        assembly = average_repetition(assembly)
+    return assembly

brainscore_vision/benchmarks/sanghavi2020/benchmarks/sanghavijozwik2020_benchmark.py

@@ -0,0 +1,44 @@
+import numpy as np
+
+from brainscore_vision import load_ceiling, load_metric, load_dataset
+from brainscore_vision.benchmark_helpers.neural_common import NeuralBenchmark, average_repetition
+from brainscore_vision.data.sanghavi2020 import BIBTEX_SANGHAVIJOZWIK
+from brainscore_vision.utils import LazyLoad
+
+VISUAL_DEGREES = 8
+NUMBER_OF_TRIALS = 37
+
+
+def _DicarloSanghaviJozwik2020Region(region):
+    assembly_repetition = LazyLoad(lambda region=region: load_assembly(average_repetitions=False, region=region))
+    assembly = LazyLoad(lambda region=region: load_assembly(average_repetitions=True, region=region))
+    metric = load_metric('pls', crossvalidation_kwargs=dict(stratification_coord=None))
+    ceiler = load_ceiling('internal_consistency')
+    return NeuralBenchmark(identifier=f'SanghaviJozwik2020.{region}-pls', version=1,
+                           assembly=assembly, similarity_metric=metric,
+                           visual_degrees=VISUAL_DEGREES, number_of_trials=NUMBER_OF_TRIALS,
+                           ceiling_func=lambda: ceiler(assembly_repetition),
+                           parent=region,
+                           bibtex=BIBTEX_SANGHAVIJOZWIK)
+
+
+def DicarloSanghaviJozwik2020V4PLS():
+    return _DicarloSanghaviJozwik2020Region('V4')
+
+
+def DicarloSanghaviJozwik2020ITPLS():
+    return _DicarloSanghaviJozwik2020Region('IT')
+
+
+def load_assembly(average_repetitions, region):
+    assembly = load_dataset(f'SanghaviJozwik2020')
+    assembly = assembly.sel(region=region)
+    assembly['region'] = 'neuroid', [region] * len(assembly['neuroid'])
+    assembly.load()
+    assembly = assembly.sel(time_bin_id=0)  # 70-170ms
+    assembly = assembly.squeeze('time_bin')
+    assert NUMBER_OF_TRIALS == len(np.unique(assembly.coords['repetition']))
+    assert VISUAL_DEGREES == assembly.attrs['image_size_degree']
+    if average_repetitions:
+        assembly = average_repetition(assembly)
+    return assembly

brainscore_vision/benchmarks/sanghavi2020/benchmarks/sanghavimurty2020_benchmark.py

@@ -0,0 +1,44 @@
+import numpy as np
+
+from brainscore_vision import load_metric, load_ceiling, load_dataset
+from brainscore_vision.benchmark_helpers.neural_common import NeuralBenchmark, average_repetition
+from brainscore_vision.data.sanghavi2020 import BIBTEX_SANGHAVIMURTY
+from brainscore_vision.utils import LazyLoad
+
+VISUAL_DEGREES = 5
+NUMBER_OF_TRIALS = 50
+
+
+def _DicarloSanghaviMurty2020Region(region):
+    assembly_repetition = LazyLoad(lambda region=region: load_assembly(average_repetitions=False, region=region))
+    assembly = LazyLoad(lambda region=region: load_assembly(average_repetitions=True, region=region))
+    metric = load_metric('pls', crossvalidation_kwargs=dict(stratification_coord=None))
+    ceiler = load_ceiling('internal_consistency')
+    return NeuralBenchmark(identifier=f'SanghaviMurty2020.{region}-pls', version=1,
+                           assembly=assembly, similarity_metric=metric,
+                           visual_degrees=VISUAL_DEGREES, number_of_trials=NUMBER_OF_TRIALS,
+                           ceiling_func=lambda: ceiler(assembly_repetition),
+                           parent=region,
+                           bibtex=BIBTEX_SANGHAVIMURTY)
+
+
+def DicarloSanghaviMurty2020V4PLS():
+    return _DicarloSanghaviMurty2020Region('V4')
+
+
+def DicarloSanghaviMurty2020ITPLS():
+    return _DicarloSanghaviMurty2020Region('IT')
+
+
+def load_assembly(average_repetitions, region):
+    assembly = load_dataset(f'SanghaviMurty2020')
+    assembly = assembly.sel(region=region)
+    assembly['region'] = 'neuroid', [region] * len(assembly['neuroid'])
+    assembly.load()
+    assembly = assembly.sel(time_bin_id=0)  # 70-170ms
+    assembly = assembly.squeeze('time_bin')
+    assert NUMBER_OF_TRIALS == len(np.unique(assembly.coords['repetition']))
+    assert VISUAL_DEGREES == assembly.attrs['image_size_degree']
+    if average_repetitions:
+        assembly = average_repetition(assembly)
+    return assembly

brainscore_vision/benchmarks/sanghavi2020/test.py

@@ -0,0 +1,83 @@
+from pathlib import Path
+
+import pytest
+from pytest import approx
+
+from brainscore_vision import benchmark_registry
+from brainscore_vision.benchmark_helpers.test_helper import StandardizedTests, PrecomputedTests, NumberOfTrialsTests
+from brainscore_vision.data_helpers import s3
+
+standardized_tests = StandardizedTests()
+precomputed_test = PrecomputedTests()
+num_trials_test = NumberOfTrialsTests()
+
+
+@pytest.mark.parametrize('benchmark', [
+    'Sanghavi2020.V4-pls',
+    'Sanghavi2020.IT-pls',
+    'SanghaviJozwik2020.V4-pls',
+    'SanghaviJozwik2020.IT-pls',
+    'SanghaviMurty2020.V4-pls',
+    'SanghaviMurty2020.IT-pls',
+])
+def test_benchmark_registry(benchmark):
+    assert benchmark in benchmark_registry
+
+
+@pytest.mark.memory_intense
+@pytest.mark.private_access
+@pytest.mark.parametrize('benchmark, expected', [
+    ('Sanghavi2020.V4-pls', approx(.8892049, abs=.001)),
+    ('Sanghavi2020.IT-pls', approx(.868293, abs=.001)),
+    ('SanghaviJozwik2020.V4-pls', approx(.9630336, abs=.001)),
+    ('SanghaviJozwik2020.IT-pls', approx(.860352, abs=.001)),
+    ('SanghaviMurty2020.V4-pls', approx(.9666086, abs=.001)),
+    ('SanghaviMurty2020.IT-pls', approx(.875714, abs=.001)),
+])
+def test_ceilings(benchmark, expected):
+    standardized_tests.ceilings_test(benchmark, expected)
+
+
+@pytest.mark.memory_intense
+@pytest.mark.private_access
+@pytest.mark.parametrize('benchmark, visual_degrees, expected', [
+    ('Sanghavi2020.V4-pls', 8, approx(.9727137, abs=.001)),
+    ('Sanghavi2020.IT-pls', 8, approx(.890062, abs=.001)),
+    ('SanghaviJozwik2020.V4-pls', 8, approx(.9739177, abs=.001)),
+    ('SanghaviJozwik2020.IT-pls', 8, approx(.9999779, abs=.001)),
+    ('SanghaviMurty2020.V4-pls', 5, approx(.978581, abs=.001)),
+    ('SanghaviMurty2020.IT-pls', 5, approx(.9997532, abs=.001)),
+])
+def test_self_regression(benchmark, visual_degrees, expected):
+    standardized_tests.self_regression_test(benchmark, visual_degrees, expected)
+
+
+@pytest.mark.memory_intense
+@pytest.mark.private_access
+@pytest.mark.slow
+@pytest.mark.parametrize('benchmark, filename, expected', [
+    ('Sanghavi2020.V4-pls', 'alexnet-sanghavi2020-features.12.nc', approx(.551135, abs=.015)),
+    ('Sanghavi2020.IT-pls', 'alexnet-sanghavi2020-features.12.nc', approx(.611347, abs=.015)),
+    ('SanghaviJozwik2020.V4-pls', 'alexnet-sanghavijozwik2020-features.12.nc', approx(.49235, abs=.005)),
+    ('SanghaviJozwik2020.IT-pls', 'alexnet-sanghavijozwik2020-features.12.nc', approx(.590543, abs=.005)),
+    ('SanghaviMurty2020.V4-pls', 'alexnet-sanghavimurty2020-features.12.nc', approx(.357461, abs=.015)),
+    ('SanghaviMurty2020.IT-pls', 'alexnet-sanghavimurty2020-features.12.nc', approx(.53006, abs=.015)),
+])
+def test_model_features(benchmark, filename, expected):
+    filepath = Path(__file__).parent / filename
+    s3.download_file_if_not_exists(local_path=filepath,
+                                   bucket='brainscore-unittests', remote_filepath=f'tests/test_benchmarks/{filename}')
+    precomputed_test.run_test(benchmark=benchmark, precomputed_features_filepath=filepath, expected=expected)
+
+
+@pytest.mark.private_access
+@pytest.mark.parametrize('benchmark_identifier', [
+    'Sanghavi2020.V4-pls',
+    'Sanghavi2020.IT-pls',
+    'SanghaviJozwik2020.V4-pls',
+    'SanghaviJozwik2020.IT-pls',
+    'SanghaviMurty2020.V4-pls',
+    'SanghaviMurty2020.IT-pls',
+])
+def test_repetitions(benchmark_identifier):
+    num_trials_test.repetitions_test(benchmark_identifier)

brainscore_vision/benchmarks/scialom2024/__init__.py

@@ -0,0 +1,52 @@
+from brainscore_vision import benchmark_registry
+from . import benchmark
+
+# behavioral benchmarks
+benchmark_registry['Scialom2024_rgbBehavioralAccuracyDistance'] = lambda: benchmark._Scialom2024BehavioralAccuracyDistance('rgb')
+benchmark_registry['Scialom2024_contoursBehavioralAccuracyDistance'] = lambda: benchmark._Scialom2024BehavioralAccuracyDistance('contours')
+benchmark_registry['Scialom2024_phosphenes-12BehavioralAccuracyDistance'] = lambda: benchmark._Scialom2024BehavioralAccuracyDistance('phosphenes-12')
+benchmark_registry['Scialom2024_phosphenes-16BehavioralAccuracyDistance'] = lambda: benchmark._Scialom2024BehavioralAccuracyDistance('phosphenes-16')
+benchmark_registry['Scialom2024_phosphenes-21BehavioralAccuracyDistance'] = lambda: benchmark._Scialom2024BehavioralAccuracyDistance('phosphenes-21')
+benchmark_registry['Scialom2024_phosphenes-27BehavioralAccuracyDistance'] = lambda: benchmark._Scialom2024BehavioralAccuracyDistance('phosphenes-27')
+benchmark_registry['Scialom2024_phosphenes-35BehavioralAccuracyDistance'] = lambda: benchmark._Scialom2024BehavioralAccuracyDistance('phosphenes-35')
+benchmark_registry['Scialom2024_phosphenes-46BehavioralAccuracyDistance'] = lambda: benchmark._Scialom2024BehavioralAccuracyDistance('phosphenes-46')
+benchmark_registry['Scialom2024_phosphenes-59BehavioralAccuracyDistance'] = lambda: benchmark._Scialom2024BehavioralAccuracyDistance('phosphenes-59')
+benchmark_registry['Scialom2024_phosphenes-77BehavioralAccuracyDistance'] = lambda: benchmark._Scialom2024BehavioralAccuracyDistance('phosphenes-77')
+benchmark_registry['Scialom2024_phosphenes-100BehavioralAccuracyDistance'] = lambda: benchmark._Scialom2024BehavioralAccuracyDistance('phosphenes-100')
+benchmark_registry['Scialom2024_segments-12BehavioralAccuracyDistance'] = lambda: benchmark._Scialom2024BehavioralAccuracyDistance('segments-12')
+benchmark_registry['Scialom2024_segments-16BehavioralAccuracyDistance'] = lambda: benchmark._Scialom2024BehavioralAccuracyDistance('segments-16')
+benchmark_registry['Scialom2024_segments-21BehavioralAccuracyDistance'] = lambda: benchmark._Scialom2024BehavioralAccuracyDistance('segments-21')
+benchmark_registry['Scialom2024_segments-27BehavioralAccuracyDistance'] = lambda: benchmark._Scialom2024BehavioralAccuracyDistance('segments-27')
+benchmark_registry['Scialom2024_segments-35BehavioralAccuracyDistance'] = lambda: benchmark._Scialom2024BehavioralAccuracyDistance('segments-35')
+benchmark_registry['Scialom2024_segments-46BehavioralAccuracyDistance'] = lambda: benchmark._Scialom2024BehavioralAccuracyDistance('segments-46')
+benchmark_registry['Scialom2024_segments-59BehavioralAccuracyDistance'] = lambda: benchmark._Scialom2024BehavioralAccuracyDistance('segments-59')
+benchmark_registry['Scialom2024_segments-77BehavioralAccuracyDistance'] = lambda: benchmark._Scialom2024BehavioralAccuracyDistance('segments-77')
+benchmark_registry['Scialom2024_segments-100BehavioralAccuracyDistance'] = lambda: benchmark._Scialom2024BehavioralAccuracyDistance('segments-100')
+
+# composites
+benchmark_registry['Scialom2024_phosphenes-allBehavioralErrorConsistency'] = lambda: benchmark._Scialom2024BehavioralErrorConsistency('phosphenes-all')
+benchmark_registry['Scialom2024_segments-allBehavioralErrorConsistency'] = lambda: benchmark._Scialom2024BehavioralErrorConsistency('segments-all')
+benchmark_registry['Scialom2024_phosphenes-allBehavioralAccuracyDistance'] = lambda: benchmark._Scialom2024BehavioralAccuracyDistance('phosphenes-all')
+benchmark_registry['Scialom2024_segments-allBehavioralAccuracyDistance'] = lambda: benchmark._Scialom2024BehavioralAccuracyDistance('segments-all')
+
+# engineering benchmarks
+benchmark_registry['Scialom2024_rgbEngineeringAccuracy'] = lambda: benchmark._Scialom2024EngineeringAccuracy('rgb')
+benchmark_registry['Scialom2024_contoursEngineeringAccuracy'] = lambda: benchmark._Scialom2024EngineeringAccuracy('contours')
+benchmark_registry['Scialom2024_phosphenes-12EngineeringAccuracy'] = lambda: benchmark._Scialom2024EngineeringAccuracy('phosphenes-12')
+benchmark_registry['Scialom2024_phosphenes-16EngineeringAccuracy'] = lambda: benchmark._Scialom2024EngineeringAccuracy('phosphenes-16')
+benchmark_registry['Scialom2024_phosphenes-21EngineeringAccuracy'] = lambda: benchmark._Scialom2024EngineeringAccuracy('phosphenes-21')
+benchmark_registry['Scialom2024_phosphenes-27EngineeringAccuracy'] = lambda: benchmark._Scialom2024EngineeringAccuracy('phosphenes-27')
+benchmark_registry['Scialom2024_phosphenes-35EngineeringAccuracy'] = lambda: benchmark._Scialom2024EngineeringAccuracy('phosphenes-35')
+benchmark_registry['Scialom2024_phosphenes-46EngineeringAccuracy'] = lambda: benchmark._Scialom2024EngineeringAccuracy('phosphenes-46')
+benchmark_registry['Scialom2024_phosphenes-59EngineeringAccuracy'] = lambda: benchmark._Scialom2024EngineeringAccuracy('phosphenes-59')
+benchmark_registry['Scialom2024_phosphenes-77EngineeringAccuracy'] = lambda: benchmark._Scialom2024EngineeringAccuracy('phosphenes-77')
+benchmark_registry['Scialom2024_phosphenes-100EngineeringAccuracy'] = lambda: benchmark._Scialom2024EngineeringAccuracy('phosphenes-100')
+benchmark_registry['Scialom2024_segments-12EngineeringAccuracy'] = lambda: benchmark._Scialom2024EngineeringAccuracy('segments-12')
+benchmark_registry['Scialom2024_segments-16EngineeringAccuracy'] = lambda: benchmark._Scialom2024EngineeringAccuracy('segments-16')
+benchmark_registry['Scialom2024_segments-21EngineeringAccuracy'] = lambda: benchmark._Scialom2024EngineeringAccuracy('segments-21')
+benchmark_registry['Scialom2024_segments-27EngineeringAccuracy'] = lambda: benchmark._Scialom2024EngineeringAccuracy('segments-27')
+benchmark_registry['Scialom2024_segments-35EngineeringAccuracy'] = lambda: benchmark._Scialom2024EngineeringAccuracy('segments-35')
+benchmark_registry['Scialom2024_segments-46EngineeringAccuracy'] = lambda: benchmark._Scialom2024EngineeringAccuracy('segments-46')
+benchmark_registry['Scialom2024_segments-59EngineeringAccuracy'] = lambda: benchmark._Scialom2024EngineeringAccuracy('segments-59')
+benchmark_registry['Scialom2024_segments-77EngineeringAccuracy'] = lambda: benchmark._Scialom2024EngineeringAccuracy('segments-77')
+benchmark_registry['Scialom2024_segments-100EngineeringAccuracy'] = lambda: benchmark._Scialom2024EngineeringAccuracy('segments-100')