brainscore-vision 2.1__py3-none-any.whl

brainscore_vision/benchmarks/geirhos2021/benchmark.py

@@ -0,0 +1,132 @@
+import numpy as np
+
+from brainio.assemblies import walk_coords
+from brainscore_vision import load_dataset, load_metric
+from brainscore_vision.benchmark_helpers.screen import place_on_screen
+from brainscore_vision.benchmarks import BenchmarkBase
+from brainscore_vision.metrics import Score
+from brainscore_vision.model_interface import BrainModel
+from brainscore_vision.utils import LazyLoad
+
+BIBTEX = """@article{geirhos2021partial,
+              title={Partial success in closing the gap between human and machine vision},
+              author={Geirhos, Robert and Narayanappa, Kantharaju and Mitzkus, Benjamin and Thieringer, Tizian and Bethge, Matthias and Wichmann, Felix A and Brendel, Wieland},
+              journal={Advances in Neural Information Processing Systems},
+              volume={34},
+              year={2021},
+              url={https://openreview.net/forum?id=QkljT4mrfs}
+        }"""
+
+DATASETS = ['colour', 'contrast', 'cue-conflict', 'edge',
+            'eidolonI', 'eidolonII', 'eidolonIII',
+            'false-colour', 'high-pass', 'low-pass', 'phase-scrambling', 'power-equalisation',
+            'rotation', 'silhouette', 'sketch', 'stylized', 'uniform-noise']
+
+# exclusion criteria: (from https://github.com/bethgelab/model-vs-human/blob/1a2dc996349cc6560bb8a98734d2882b3b308585/modelvshuman/plotting/plot.py#L43)
+# - not OOD: control condition without manipulation (e.g. 100% contrast)
+# - mean human accuracy < 0.2 (error consistency etc. not meaningful)
+EXCLUDE_CONDITIONS = {
+    "colour": ["cr"],
+    "contrast": ["c100", "c03", "c01"],
+    "high-pass": [np.inf, 0.55, 0.45, 0.4],
+    "low-pass": [0, 15, 40],
+    "phase-scrambling": [0, 150, 180],
+    "power-equalisation": ["0"],
+    "false-colour": ["False-Colour"],
+    "rotation": [0],
+    "eidolonI": ["1-10-10", "64-10-10", "128-10-10"],
+    "eidolonII": ["1-3-10", "32-3-10", "64-3-10", "128-3-10"],
+    "eidolonIII": ["1-0-10", "16-0-10", "32-0-10", "64-0-10", "128-0-10"],
+    "uniform-noise": [0.0, 0.6, 0.9]
+}
+
+# create functions so that users can import individual benchmarks as e.g. Geirhos2021sketchErrorConsistency
+for dataset in DATASETS:
+    # behavioral benchmark
+    identifier = f"Geirhos2021{dataset.replace('-', '')}ErrorConsistency"
+    globals()[identifier] = lambda dataset=dataset: _Geirhos2021ErrorConsistency(dataset)
+    # engineering benchmark
+    identifier = f"Geirhos2021{dataset.replace('-', '')}Accuracy"
+    globals()[identifier] = lambda dataset=dataset: _Geirhos2021Accuracy(dataset)
+
+
+class _Geirhos2021ErrorConsistency(BenchmarkBase):
+    # behavioral benchmark
+    def __init__(self, dataset):
+        self._metric = load_metric('error_consistency')
+        self._assembly = LazyLoad(lambda: load_assembly(dataset))
+        self._visual_degrees = 3
+
+        self._number_of_trials = 1
+
+        dataset_nodashes = dataset.replace("-", "")
+        super(_Geirhos2021ErrorConsistency, self).__init__(
+            identifier=f'Geirhos2021{dataset_nodashes}-error_consistency', version=1,
+            ceiling_func=lambda: self._metric.ceiling(self._assembly),
+            parent='Geirhos2021',
+            bibtex=BIBTEX)
+
+    def __call__(self, candidate: BrainModel):
+        choice_labels = set(self._assembly['truth'].values)
+        choice_labels = list(sorted(choice_labels))
+        candidate.start_task(BrainModel.Task.label, choice_labels)
+        stimulus_set = place_on_screen(self._assembly.stimulus_set, target_visual_degrees=candidate.visual_degrees(),
+                                       source_visual_degrees=self._visual_degrees)
+        labels = candidate.look_at(stimulus_set, number_of_trials=self._number_of_trials)
+        raw_score = self._metric(labels, self._assembly)
+        ceiling = self.ceiling
+        score = raw_score / ceiling
+        score.attrs['raw'] = raw_score
+        score.attrs['ceiling'] = ceiling
+        return score
+
+
+class _Geirhos2021Accuracy(BenchmarkBase):
+    # engineering/ML benchmark
+    def __init__(self, dataset):
+        self._metric = load_metric('accuracy')
+        self._stimulus_set = LazyLoad(lambda: load_assembly(dataset).stimulus_set)
+        dataset_nodashes = dataset.replace("-", "")
+        super(_Geirhos2021Accuracy, self).__init__(
+            identifier=f'Geirhos2021{dataset_nodashes}-top1', version=1,
+            ceiling_func=lambda: Score(1),
+            parent='Geirhos2021-top1',
+            bibtex=BIBTEX)
+
+    def __call__(self, candidate: BrainModel):
+        choice_labels = set(self._stimulus_set['truth'].values)
+        choice_labels = list(sorted(choice_labels))
+        candidate.start_task(BrainModel.Task.label, choice_labels)
+        labels = candidate.look_at(self._stimulus_set, number_of_trials=10)
+        score = self._metric(labels, target=self._stimulus_set['truth'].values)
+        return score
+
+
+def load_assembly(dataset):
+    assembly = load_dataset(f'Geirhos2021_{dataset}')
+
+    # exclude conditions following the paper
+    if dataset in EXCLUDE_CONDITIONS:
+        excluded = EXCLUDE_CONDITIONS[dataset]
+        assert all(condition in assembly['condition'].values for condition in excluded)
+        assembly = assembly[{'presentation': ~np.isin(assembly['condition'], excluded)}]
+        stimulus_set = assembly.attrs['stimulus_set']
+        stimulus_set = stimulus_set[stimulus_set['image_id'].isin(set(assembly['image_id'].values))]
+        assembly.attrs['stimulus_set'] = stimulus_set
+    # convert condition float to string to avoid xarray indexing errors.
+    # See https://app.travis-ci.com/github/brain-score/brain-score/builds/256059224
+    assembly = cast_coordinate_type(assembly, coordinate='condition', newtype=str)
+    assembly.attrs['stimulus_set']['condition'] = assembly.attrs['stimulus_set']['condition'].astype(str)
+    return assembly
+
+
+def cast_coordinate_type(assembly, coordinate, newtype):
+    attrs = assembly.attrs
+    condition_values = assembly[coordinate].values
+    assembly = type(assembly)(assembly.values, coords={
+        coord: (dims, values) for coord, dims, values in walk_coords(assembly) if coord != coordinate},
+                              dims=assembly.dims)
+    assembly[coordinate] = 'presentation', condition_values.astype(newtype)
+    assembly = type(assembly)(assembly)
+    assembly.attrs = attrs
+    return assembly

brainscore_vision/benchmarks/geirhos2021/test.py

@@ -0,0 +1,189 @@
+from pathlib import Path
+
+import numpy as np
+import pytest
+from pytest import approx
+
+from brainio.assemblies import BehavioralAssembly
+from brainscore_vision import benchmark_registry, load_benchmark
+from brainscore_vision.benchmark_helpers import PrecomputedFeatures
+from brainscore_vision.benchmarks.geirhos2021.benchmark import DATASETS, cast_coordinate_type
+from brainscore_vision.data_helpers import s3
+
+
+@pytest.mark.parametrize('benchmark', [
+    'Geirhos2021colour-error_consistency',
+    'Geirhos2021contrast-error_consistency',
+    'Geirhos2021cueconflict-error_consistency',
+    'Geirhos2021edge-error_consistency',
+    'Geirhos2021eidolonI-error_consistency',
+    'Geirhos2021eidolonII-error_consistency',
+    'Geirhos2021eidolonIII-error_consistency',
+    'Geirhos2021falsecolour-error_consistency',
+    'Geirhos2021highpass-error_consistency',
+    'Geirhos2021lowpass-error_consistency',
+    'Geirhos2021phasescrambling-error_consistency',
+    'Geirhos2021powerequalisation-error_consistency',
+    'Geirhos2021rotation-error_consistency',
+    'Geirhos2021silhouette-error_consistency',
+    'Geirhos2021stylized-error_consistency',
+    'Geirhos2021sketch-error_consistency',
+    'Geirhos2021uniformnoise-error_consistency',
+    'Geirhos2021colour-top1',
+    'Geirhos2021contrast-top1',
+    'Geirhos2021cueconflict-top1',
+    'Geirhos2021edge-top1',
+    'Geirhos2021eidolonI-top1',
+    'Geirhos2021eidolonII-top1',
+    'Geirhos2021eidolonIII-top1',
+    'Geirhos2021falsecolour-top1',
+    'Geirhos2021highpass-top1',
+    'Geirhos2021lowpass-top1',
+    'Geirhos2021phasescrambling-top1',
+    'Geirhos2021powerequalisation-top1',
+    'Geirhos2021rotation-top1',
+    'Geirhos2021silhouette-top1',
+    'Geirhos2021stylized-top1',
+    'Geirhos2021sketch-top1',
+    'Geirhos2021uniformnoise-top1',
+])
+def test_benchmark_registry(benchmark):
+    assert benchmark in benchmark_registry
+
+
+class TestBehavioral:
+    def test_count(self):
+        assert len(DATASETS) == 12 + 5
+
+    @pytest.mark.parametrize('dataset', DATASETS)
+    def test_in_pool(self, dataset):
+        identifier = f"Geirhos2021{dataset.replace('-', '')}-error_consistency"
+        assert identifier in benchmark_registry
+
+    def test_mean_ceiling(self):
+        benchmarks = [f"Geirhos2021{dataset.replace('-', '')}-error_consistency" for dataset in DATASETS]
+        benchmarks = [load_benchmark(benchmark) for benchmark in benchmarks]
+        ceilings = [benchmark.ceiling.item() for benchmark in benchmarks]
+        mean_ceiling = np.mean(ceilings)
+        assert mean_ceiling == approx(0.43122, abs=0.001)
+
+    @pytest.mark.parametrize('dataset, expected_ceiling', [
+        ('colour', approx(0.41543, abs=0.001)),
+        ('contrast', approx(0.43703, abs=0.001)),
+        ('cue-conflict', approx(0.33105, abs=0.001)),
+        ('edge', approx(0.31844, abs=0.001)),
+        ('eidolonI', approx(0.38634, abs=0.001)),
+        ('eidolonII', approx(0.45402, abs=0.001)),
+        ('eidolonIII', approx(0.45953, abs=0.001)),
+        ('false-colour', approx(0.44405, abs=0.001)),
+        ('high-pass', approx(0.44014, abs=0.001)),
+        ('low-pass', approx(0.46888, abs=0.001)),
+        ('phase-scrambling', approx(0.44667, abs=0.001)),
+        ('power-equalisation', approx(0.51063, abs=0.001)),
+        ('rotation', approx(0.43851, abs=0.001)),
+        ('silhouette', approx(0.47571, abs=0.001)),
+        ('sketch', approx(0.36962, abs=0.001)),
+        ('stylized', approx(0.50058, abs=0.001)),
+        ('uniform-noise', approx(0.43406, abs=0.001)),
+    ])
+    def test_dataset_ceiling(self, dataset, expected_ceiling):
+        benchmark = f"Geirhos2021{dataset.replace('-', '')}-error_consistency"
+        benchmark = load_benchmark(benchmark)
+        ceiling = benchmark.ceiling
+        assert ceiling == expected_ceiling
+
+    @pytest.mark.parametrize('dataset, expected_raw_score', [
+        ('colour', approx(0.21135, abs=0.001)),
+        ('contrast', approx(0.22546, abs=0.001)),
+        ('cue-conflict', approx(0.06800, abs=0.001)),
+        ('edge', approx(0.03626, abs=0.001)),
+        ('eidolonI', approx(0.21029, abs=0.001)),
+        ('eidolonII', approx(0.25090, abs=0.001)),
+        ('eidolonIII', approx(0.15806, abs=0.001)),
+        ('false-colour', approx(0.29431, abs=0.001)),
+        ('high-pass', approx(0.20300, abs=0.001)),
+        ('low-pass', approx(0.16741, abs=0.001)),
+        ('phase-scrambling', approx(0.26879, abs=0.001)),
+        ('power-equalisation', approx(0.29293, abs=0.001)),
+        ('rotation', approx(0.16892, abs=0.001)),
+        ('silhouette', approx(0.42805, abs=0.001)),
+        ('sketch', approx(0.10537, abs=0.001)),
+        ('stylized', approx(0.25430, abs=0.001)),
+        ('uniform-noise', approx(0.19839, abs=0.001)),
+    ])
+    def test_model_3degrees(self, dataset, expected_raw_score):
+        benchmark = load_benchmark(f"Geirhos2021{dataset.replace('-', '')}-error_consistency")
+        # load features
+        filename = f'resnet-50-pytorch-3deg-Geirhos2021_{dataset}.nc'
+        precomputed_features = Path(__file__).parent / filename
+        s3.download_file_if_not_exists(precomputed_features,
+                                       bucket='brain-score-tests', remote_filepath=f'tests/test_benchmarks/{filename}')
+        precomputed_features = BehavioralAssembly.from_files(file_path=precomputed_features)
+        # these features were packaged with condition as int/float. Current xarray versions have trouble when
+        # selecting for a float coordinate however, so we had to change the type to string.
+        precomputed_features = cast_coordinate_type(precomputed_features, 'condition', newtype=str)
+        precomputed_features = PrecomputedFeatures(precomputed_features, visual_degrees=8)
+        # score
+        score = benchmark(precomputed_features)
+        raw_score = score.raw
+        # division by ceiling <= 1 should result in higher score
+        assert score >= raw_score
+        assert raw_score == expected_raw_score
+
+    @pytest.mark.parametrize('model, expected_raw_score', [
+        ('resnet-50-pytorch-3deg', approx(0.20834, abs=0.001)),
+        ('resnet-50-pytorch-8deg', approx(0.10256, abs=0.001)),
+    ])
+    def test_model_mean(self, model, expected_raw_score):
+        scores = []
+        for dataset in DATASETS:
+            benchmark = load_benchmark(f"Geirhos2021{dataset.replace('-', '')}-error_consistency")
+            filename = f'{model}-Geirhos2021_{dataset}.nc'
+            precomputed_features = Path(__file__).parent / filename
+            s3.download_file_if_not_exists(precomputed_features,
+                                           bucket='brain-score-tests',
+                                           remote_filepath=f'tests/test_benchmarks/{filename}')
+            precomputed_features = BehavioralAssembly.from_files(file_path=precomputed_features)
+            # these features were packaged with condition as int/float. Current xarray versions have trouble when
+            # selecting for a float coordinate however, so we had to change the type to string.
+            precomputed_features = cast_coordinate_type(precomputed_features, 'condition', newtype=str)
+            precomputed_features = PrecomputedFeatures(precomputed_features, visual_degrees=8)
+            score = benchmark(precomputed_features).raw
+            scores.append(score.item())
+        mean_score = np.mean(scores)
+        assert mean_score == expected_raw_score
+
+
+class TestEngineering:
+    @pytest.mark.private_access
+    @pytest.mark.parametrize('dataset, model, expected_accuracy', [
+        ('colour', 'resnet-50-pytorch', approx(0.96875, abs=0.001)),
+        ('contrast', 'resnet-50-pytorch', approx(0.81625, abs=0.001)),
+        ('cue-conflict', 'resnet-50-pytorch', approx(0.18203, abs=0.001)),
+        ('edge', 'resnet-50-pytorch', approx(0.13750, abs=0.001)),
+        ('eidolonI', 'resnet-50-pytorch', approx(0.53375, abs=0.001)),
+        ('eidolonII', 'resnet-50-pytorch', approx(0.56719, abs=0.001)),
+        ('eidolonIII', 'resnet-50-pytorch', approx(0.58542, abs=0.001)),
+        ('false-colour', 'resnet-50-pytorch', approx(0.94286, abs=0.001)),
+        ('high-pass', 'resnet-50-pytorch', approx(0.33437, abs=0.001)),
+        ('low-pass', 'resnet-50-pytorch', approx(0.43875, abs=0.001)),
+        ('phase-scrambling', 'resnet-50-pytorch', approx(0.62031, abs=0.001)),
+        ('power-equalisation', 'resnet-50-pytorch', approx(0.73214, abs=0.001)),
+        ('rotation', 'resnet-50-pytorch', approx(0.68438, abs=0.001)),
+        ('silhouette', 'resnet-50-pytorch', approx(0.54375, abs=0.001)),
+        ('sketch', 'resnet-50-pytorch', approx(0.59625, abs=0.001)),
+        ('stylized', 'resnet-50-pytorch', approx(0.37125, abs=0.001)),
+        ('uniform-noise', 'resnet-50-pytorch', approx(0.44500, abs=0.001)),
+    ])
+    def test_accuracy(self, dataset, model, expected_accuracy):
+        benchmark = load_benchmark(f"Geirhos2021{dataset.replace('-', '')}-top1")
+        # load features
+        filename = f'{model}-3deg-Geirhos2021_{dataset}.nc'
+        precomputed_features = Path(__file__).parent / filename
+        s3.download_file_if_not_exists(precomputed_features,
+                                       bucket='brain-score-tests', remote_filepath=f'tests/test_benchmarks/{filename}')
+        precomputed_features = BehavioralAssembly.from_files(file_path=precomputed_features)
+        precomputed_features = PrecomputedFeatures(precomputed_features, visual_degrees=None)
+        # score
+        score = benchmark(precomputed_features)
+        assert score == expected_accuracy

brainscore_vision/benchmarks/hebart2023/__init__.py

@@ -0,0 +1,4 @@
+from brainscore_vision import benchmark_registry
+from .benchmark import Hebart2023Match
+
+benchmark_registry['Hebart2023-match'] = Hebart2023Match

brainscore_vision/benchmarks/hebart2023/benchmark.py

@@ -0,0 +1,72 @@
+import numpy as np
+
+from brainscore_vision import load_dataset, load_stimulus_set
+from brainscore_vision.benchmark_helpers import bound_score
+from brainscore_vision.benchmark_helpers.screen import place_on_screen
+from brainscore_vision.benchmarks import BenchmarkBase
+from brainscore_vision.metrics import Score
+from brainscore_vision.model_interface import BrainModel
+
+BIBTEX = """@article{10.7554/eLife.82580,
+          author = {Hebart, Martin N and Contier, Oliver and Teichmann, Lina and Rockter, Adam H and Zheng, Charles Y and Kidder, Alexis and Corriveau, Anna and Vaziri-Pashkam, Maryam and Baker, Chris I},
+          journal = {eLife},
+          month = {feb},
+          pages = {e82580},
+          title = {THINGS-data, a multimodal collection of large-scale datasets for investigating object representations in human brain and behavior},
+          volume = 12,
+          year = 2023
+          }"""
+VISUAL_DEGREES = 8
+
+
+class Hebart2023Match(BenchmarkBase):
+    def __init__(self):
+        self._visual_degrees = VISUAL_DEGREES
+        self._number_of_trials = 1
+        self._assembly = load_dataset('Hebart2023')
+        self._stimulus_set = load_stimulus_set('Hebart2023')
+
+        # The noise ceiling was computed by averaging the percentage of participants 
+        # who made the same choice for a given triplet. See the paper for more detail.
+        super().__init__(
+            identifier='Hebart2023-match', version=1,
+            ceiling_func=lambda: Score(0.6767),
+            parent='behavior_vision',
+            bibtex=BIBTEX
+        )
+
+    def set_number_of_triplets(self, n: int):
+        """ Allows to reduce the number of triplets to reduce the compute requirements for debugging """
+        self._assembly = self._assembly[:n]
+
+    def __call__(self, candidate: BrainModel):
+        # Create stimuli with triplets (all 3 consecutive stimuli form one trial following model_interface)
+        self.triplets = np.array([
+            self._assembly.coords["image_1"].values,
+            self._assembly.coords["image_2"].values,
+            self._assembly.coords["image_3"].values
+        ]).T.reshape(-1)  # flatten into a list of stimuli ids
+
+        triplet_stimuli = self._stimulus_set.loc[self.triplets]
+
+        # Prepare the stimuli
+        candidate.start_task(BrainModel.Task.odd_one_out)
+        triplet_stimuli = place_on_screen(
+            stimulus_set=triplet_stimuli,
+            target_visual_degrees=candidate.visual_degrees(),
+            source_visual_degrees=self._visual_degrees
+        )
+
+        # Run the model
+        choices = candidate.look_at(triplet_stimuli, self._number_of_trials)
+
+        # Score the model
+        # We chose not to compute error estimates but you could compute them
+        # by spliting the data into five folds and computing the standard deviation.
+        correct_choices = choices.values == self._assembly.coords["image_3"].values  # third image is always correct
+        raw_score = np.sum(correct_choices) / len(choices['presentation'])
+        score = (raw_score - 1 / 3) / (self.ceiling - 1 / 3)
+        bound_score(score)
+        score.attrs['raw'] = raw_score
+        score.attrs['ceiling'] = self.ceiling
+        return score

brainscore_vision/benchmarks/hebart2023/test.py

@@ -0,0 +1,19 @@
+import pytest
+
+from brainscore_vision import load_benchmark, load_model
+
+
+@pytest.mark.private_access
+def test_ceiling():
+    benchmark = load_benchmark('Hebart2023-match')
+    ceiling = benchmark.ceiling
+    assert ceiling == pytest.approx(0.6767, abs=0.0001)
+
+
+@pytest.mark.private_access
+def test_alexnet_consistency():
+    benchmark = load_benchmark('Hebart2023-match')
+    benchmark.set_number_of_triplets(n=1000)  # reduce number for faster running test
+    model = load_model('alexnet')
+    score = benchmark(model)
+    assert score == pytest.approx(0.38, abs=0.02)

brainscore_vision/benchmarks/hermann2020/__init__.py

@@ -0,0 +1,6 @@
+from brainscore_vision import benchmark_registry
+from .benchmark import Hermann2020cueconflictShapeBias, Hermann2020cueconflictShapeMatch
+
+
+benchmark_registry['Hermann2020cueconflict-shape_bias'] = Hermann2020cueconflictShapeBias
+benchmark_registry['Hermann2020cueconflict-shape_match'] = Hermann2020cueconflictShapeMatch

brainscore_vision/benchmarks/hermann2020/benchmark.py

@@ -0,0 +1,63 @@
+from brainscore_core import Score
+
+from brainscore_vision import load_stimulus_set, load_metric
+from brainscore_vision.benchmarks import BenchmarkBase
+from brainscore_vision.model_interface import BrainModel
+
+BIBTEX = """@article{hermann2020origins,
+              title={The origins and prevalence of texture bias in convolutional neural networks},
+              author={Hermann, Katherine and Chen, Ting and Kornblith, Simon},
+              journal={Advances in Neural Information Processing Systems},
+              volume={33},
+              pages={19000--19015},
+              year={2020},
+              url={https://proceedings.neurips.cc/paper/2020/hash/db5f9f42a7157abe65bb145000b5871a-Abstract.html}
+        }"""
+
+
+class _Hermann2020Match(BenchmarkBase):
+    def __init__(self, metric_identifier, stimulus_column):
+        assert metric_identifier in ["shape_match", "texture_match"]
+        assert stimulus_column in ["original_image_category", "conflict_image_category"]
+        self._stimulus_set = load_stimulus_set("Geirhos2021_cue-conflict")
+        self._metric = load_metric('accuracy')
+        self._number_of_trials = 1
+        self._stimulus_column = stimulus_column
+        super(_Hermann2020Match, self).__init__(
+            identifier=f'Hermann2020cueconflict-{metric_identifier}', version=1,
+            ceiling_func=lambda: Score(1),
+            parent='Hermann2020',
+            bibtex=BIBTEX)
+
+    def __call__(self, candidate: BrainModel):
+        choice_labels = set(self._stimulus_set['truth'].values)
+        choice_labels = list(sorted(choice_labels))
+        candidate.start_task(BrainModel.Task.label, choice_labels)
+        labels = candidate.look_at(self._stimulus_set, number_of_trials=self._number_of_trials)
+        # remove entries without cue-conflict
+        labels = labels[0][labels["conflict_image_category"] != labels["original_image_category"]]
+        cueconflict = self._stimulus_set["conflict_image_category"] != self._stimulus_set["original_image_category"]
+        target = self._stimulus_set[cueconflict]
+        target = target[self._stimulus_column].values
+        score = self._metric(labels, target)
+        return score
+
+
+class Hermann2020cueconflictShapeBias(BenchmarkBase):
+    def __init__(self):
+        self.shape_benchmark = _Hermann2020Match("shape_match", "original_image_category")
+        self.texture_benchmark = _Hermann2020Match("texture_match", "conflict_image_category")
+        super(Hermann2020cueconflictShapeBias, self).__init__(
+            identifier=f'Hermann2020cueconflict-shape_bias', version=1,
+            ceiling_func=lambda: Score(1),
+            parent='Hermann2020',
+            bibtex=BIBTEX)
+
+    def __call__(self, candidate: BrainModel):
+        shape_match = self.shape_benchmark(candidate)
+        texture_match = self.texture_benchmark(candidate)
+        return shape_match / (shape_match + texture_match)
+
+
+def Hermann2020cueconflictShapeMatch():
+    return _Hermann2020Match("shape_match", "original_image_category")

brainscore_vision/benchmarks/hermann2020/test.py

@@ -0,0 +1,28 @@
+from pathlib import Path
+
+import pytest
+from pytest import approx
+
+from brainio.assemblies import BehavioralAssembly
+from brainscore_vision import load_benchmark
+from brainscore_vision.benchmark_helpers import PrecomputedFeatures
+from brainscore_vision.data_helpers import s3
+
+
+class TestEngineering:
+    @pytest.mark.parametrize('model, benchmark, expected_shape_bias', [
+        ('resnet-50-pytorch', "Hermann2020cueconflict-shape_bias", approx(0.21392405, abs=0.001)),
+        ('resnet-50-pytorch', "Hermann2020cueconflict-shape_match", approx(0.14083333, abs=0.001)),
+    ])
+    def test_score(self, model, benchmark, expected_shape_bias):
+        benchmark = load_benchmark(benchmark)
+        # load features
+        filename = f'{model}-3deg-Geirhos2021_cue-conflict.nc'
+        filepath = Path(__file__).parent / filename
+        s3.download_file_if_not_exists(local_path=filepath,
+                                       bucket='brain-score-tests', remote_filepath=f'tests/test_benchmarks/{filename}')
+        precomputed_features = BehavioralAssembly.from_files(file_path=filepath)
+        precomputed_features = PrecomputedFeatures(precomputed_features, visual_degrees=None)
+        # score
+        score = benchmark(precomputed_features)
+        assert score == expected_shape_bias

brainscore_vision/benchmarks/igustibagus2024/__init__.py

@@ -0,0 +1,11 @@
+from brainscore_vision import benchmark_registry
+
+# neural alignment
+from .domain_transfer_neural import Igustibagus2024_ridge
+
+benchmark_registry['Igustibagus2024-ridge'] = Igustibagus2024_ridge
+
+# analysis benchmarks
+from .domain_transfer_analysis import OOD_AnalysisBenchmark
+
+benchmark_registry['Igustibagus2024.IT_readout-accuracy'] = OOD_AnalysisBenchmark