brainscore-vision 2.1__py3-none-any.whl

brainscore_vision/benchmark_helpers/test_helper.py

@@ -0,0 +1,160 @@
+from pathlib import Path
+from typing import List, Tuple
+
+import numpy as np
+from PIL import Image
+
+from brainio.assemblies import NeuroidAssembly, PropertyAssembly
+from brainscore_vision import load_benchmark
+from brainscore_vision.model_interface import BrainModel
+from brainscore_vision.data_helpers import s3
+from . import PrecomputedFeatures
+
+
+class StandardizedTests:
+    def ceilings_test(self, benchmark: str, expected: float):
+        benchmark = load_benchmark(benchmark)
+        ceiling = benchmark.ceiling
+        assert ceiling == expected
+
+    def self_regression_test(self, benchmark: str, visual_degrees: int, expected: float):
+        benchmark = load_benchmark(benchmark)
+        score = benchmark(PrecomputedFeatures(benchmark._assembly, visual_degrees=visual_degrees)).raw
+        assert score == expected
+        raw_values = score.attrs['raw']
+        assert hasattr(raw_values, 'neuroid')
+        assert hasattr(raw_values, 'split')
+        assert len(raw_values['split']) == 10
+
+    def self_rdm_test(self, benchmark: str, visual_degrees: int, expected: float):
+        benchmark = load_benchmark(benchmark)
+        score = benchmark(PrecomputedFeatures(benchmark._assembly, visual_degrees=visual_degrees)).raw
+        assert score == expected
+        raw_values = score.attrs['raw']
+        assert hasattr(raw_values, 'split')
+        assert len(raw_values['split']) == 10
+
+
+class PrecomputedTests:
+    def run_test(self, benchmark: str, precomputed_features_filepath: str, expected: float):
+        benchmark = load_benchmark(benchmark)
+        precomputed_features = NeuroidAssembly.from_files(
+            precomputed_features_filepath,
+            stimulus_set_identifier=benchmark._assembly.stimulus_set.identifier,
+            stimulus_set=None)
+        precomputed_features = precomputed_features.stack(presentation=['stimulus_path'])
+        precomputed_paths = list(map(lambda f: Path(f).name, precomputed_features['stimulus_path'].values))
+        # attach stimulus set meta
+        stimulus_set = benchmark._assembly.stimulus_set
+        expected_stimulus_paths = [stimulus_set.get_stimulus(image_id) for image_id in stimulus_set['stimulus_id']]
+        expected_stimulus_paths = list(map(lambda f: Path(f).name, expected_stimulus_paths))
+        assert set(precomputed_paths) == set(expected_stimulus_paths)
+        for column in stimulus_set.columns:
+            precomputed_features[column] = 'presentation', stimulus_set[column].values
+        precomputed_features = PrecomputedFeatures(precomputed_features,
+                                                   visual_degrees=10,  # doesn't matter, features are already computed
+                                                   )
+        # score
+        score = benchmark(precomputed_features).raw
+        assert score == expected
+
+    def run_test_properties(self, benchmark: str, files: dict, expected: float):
+        benchmark = load_benchmark(benchmark)
+        from brainscore_vision import load_stimulus_set
+
+        stimulus_identifiers = np.unique(np.array(['Marques2020_blank', 'Marques2020_receptive_field',
+                                                   'Marques2020_orientation',
+                                                   benchmark._assembly.stimulus_set.identifier]))
+        precomputed_features = {}
+        for current_stimulus in stimulus_identifiers:
+            stimulus_set = load_stimulus_set(current_stimulus)
+            path = Path(__file__).parent / files[current_stimulus]
+            s3.download_file_if_not_exists(local_path=path,
+                               bucket='brainscore-unittests', remote_filepath=f'tests/test_benchmarks/{files[current_stimulus]}')
+            features = PropertyAssembly.from_files(path,
+                                                   stimulus_set_identifier=stimulus_set.identifier,
+                                                   stimulus_set=stimulus_set)
+            features = features.stack(presentation=['stimulus_path'])
+            precomputed_features[current_stimulus] = features
+            precomputed_paths = [Path(f).name for f in precomputed_features[current_stimulus]['stimulus_path'].values]
+            # attach stimulus set meta
+            expected_stimulus_paths = [stimulus_set.get_stimulus(stimulus_id)
+                                       for stimulus_id in stimulus_set['stimulus_id']]
+            expected_stimulus_paths = list(map(lambda f: Path(f).name, expected_stimulus_paths))
+            assert set(precomputed_paths) == set(expected_stimulus_paths)
+            for column in stimulus_set.columns:
+                precomputed_features[current_stimulus][column] = 'presentation', stimulus_set[column].values
+
+        precomputed_features = PrecomputedFeatures(precomputed_features, visual_degrees=8)
+        # score
+        score = benchmark(precomputed_features).raw
+        assert score == expected
+
+
+class VisualDegreesTests:
+    def amount_gray_test(self, benchmark: str, candidate_degrees: int, image_id: str, expected: float):
+        benchmark = load_benchmark(benchmark)
+
+        class DummyCandidate(BrainModel):
+            class StopException(Exception):
+                pass
+
+            def visual_degrees(self):
+                return candidate_degrees
+
+            def look_at(self, stimuli, number_of_trials=1):
+                image = stimuli.get_stimulus(image_id)
+                image = Image.open(image)
+                image = np.array(image)
+                amount_gray = 0
+                for index in np.ndindex(image.shape[:2]):
+                    color = image[index]
+                    gray = [128, 128, 128]
+                    if (color == gray).all():
+                        amount_gray += 1
+                assert amount_gray / image.size == expected
+                raise self.StopException()
+
+            def start_task(self, task: BrainModel.Task, fitting_stimuli):
+                pass
+
+            def start_recording(self, recording_target: BrainModel.RecordingTarget, time_bins=List[Tuple[int]]):
+                pass
+
+        candidate = DummyCandidate()
+        try:
+            benchmark(candidate)  # just call to get the stimuli
+        except DummyCandidate.StopException:  # but stop early
+            pass
+
+
+class NumberOfTrialsTests:
+    def repetitions_test(self, benchmark_identifier: str):
+        """ Tests that benchmarks have repetitions in the stimulus_set """
+        benchmark = load_benchmark(benchmark_identifier)
+
+        class AssertRepeatCandidate(BrainModel):
+            class StopException(Exception):
+                pass
+
+            def identifier(self) -> str:
+                return 'assert-repeat-candidate'
+
+            def visual_degrees(self):
+                return 8
+
+            def look_at(self, stimuli, number_of_trials=1):
+                assert number_of_trials > 1
+                raise self.StopException()
+
+            def start_task(self, task: BrainModel.Task, fitting_stimuli):
+                pass
+
+            def start_recording(self, recording_target: BrainModel.RecordingTarget, time_bins=List[Tuple[int]]):
+                pass
+
+        candidate = AssertRepeatCandidate()
+        try:
+            benchmark(candidate)  # just call to get the stimuli
+        except AssertRepeatCandidate.StopException:  # but stop early
+            pass

brainscore_vision/benchmarks/README.md

@@ -0,0 +1,7 @@
+# Brain-Score Benchmarks
+
+**File structure**: One file per assembly with all the benchmarks inside.
+                    This file contains the loading of the assembly (with eventual processing) and the benchmarks.
+
+**Versioning**: Each benchmark is numbered by a version number. 
+                Whenever the benchmark produces different scores, the version number is increased.

brainscore_vision/benchmarks/__init__.py

@@ -0,0 +1,122 @@
+# Importing individual benchmarks directly is discouraged.
+# Use `brainscore_vision.load_benchmark` instead which provides dependency support.
+
+"""
+A :class:`~brainscore.benchmarks.Benchmark` runs an experiment on a :class:`~brainscore.model_interface.BrainModel`
+and tests the resulting measurements against primate `data <https://github.com/brain-score/brainio>`_.
+This comparison is done by a :class:`~brainscore.metrics.Metric` which outputs a score of how well model and data match.
+This score is normalized with data ceilings and the benchmark returns this ceiled score.
+"""
+from abc import ABC
+
+from result_caching import store
+
+from brainscore_core.metrics import Score
+from brainscore_vision.model_interface import BrainModel
+
+
+class Benchmark(ABC):
+    """
+    Standard Benchmark interface defining the method interfaces.
+    """
+
+    def __call__(self, candidate: BrainModel) -> Score:
+        """
+        Evaluate a candidate `BrainModel` and return a :class:`~brainscore.metrics.Score` denoting the brain-likeness of
+        the model under this benchmark. Typically this involves reproducing the experiment on the model and then
+        comparing model measurements (e.g. neural/behavioral) against recordings from biological subjects (e.g.
+        primates) using a :class:`~brainscore.metrics.Metric`. The output of this method is a normalized score between 0
+        and 1 where 0 means the model does not match the measurements at all and 1 means the model matches the
+        measurements at ceiling level (e.g. if the model obtains a score of 0.8 and the data ceiling is also 0.8, the
+        score output by this method should be 1).
+
+        :param candidate: a candidate model implementing the `BrainModel` interface. Benchmarks are agnostic of the
+                exact implementation and only interact with models through the methods defined in the interface.
+        :return: a :class:`~brainscore.metrics.Score` of how brain-like the candidate model is under this benchmark. The
+                score is normalized by this benchmark's ceiling such that 1 means the model matches the data to ceiling
+                level.
+        """
+        raise NotImplementedError()
+
+    @property
+    def bibtex(self) -> str:
+        """
+        bibtex string to build the reference.
+        Should include an `url` to build a proper link.
+        """
+        raise NotImplementedError()
+
+    @property
+    def identifier(self) -> str:
+        """
+        Unique identifier for this benchmark.
+        Standard format is `<data identifier>-<metric identifier>`, e.g. `Rajalingham2018-i2n`.
+
+        :return: a unique identifier for this benchmark
+        """
+        raise NotImplementedError()
+
+    @property
+    def version(self) -> str:
+        """
+        Optional, but strongly encouraged.
+
+        :return: a version number that is increased every time the model scores for this benchmark change
+                (but not for code changes that do not change scores).
+        """
+        raise NotImplementedError()
+
+    @property
+    def ceiling(self) -> Score:
+        """
+        The ceiling of this benchmark. Scores need to be normalized by this value.
+        Typically this represents the signal in the data and how well we expect the best possible model to score.
+
+        :return: a Score object, denoting the ceiling of this benchmark.
+                The Score values itself typically consist of just a scalar between zero and one.
+                Many ceilers also include the error estimate and raw values,
+                available in `ceiling.attrs['error']` and `ceiling.attrs['raw']` respectively.
+        """
+        raise NotImplementedError()
+
+
+class BenchmarkBase(Benchmark):
+    """
+    Helper class for implementing standard functions of the `Benchmark` interface.
+    """
+
+    def __init__(self, identifier, ceiling_func, version, parent=None, bibtex=None):
+        self._identifier = identifier
+        self._ceiling_func = ceiling_func
+        self._version = version
+        self.parent = parent
+        self._bibtex = bibtex
+
+    @property
+    def bibtex(self):
+        return self._bibtex
+
+    @property
+    def identifier(self):
+        return self._identifier
+
+    @property
+    def version(self):
+        return self._version
+
+    @property
+    def ceiling(self):
+        return self._ceiling(identifier=self.identifier)
+
+    @store()
+    def _ceiling(self, identifier):
+        return self._ceiling_func()
+
+
+def ceil_score(score: Score, ceiling: Score) -> Score:
+    ceiled_score = score / ceiling
+    if 'error' in score.attrs:
+        ceiled_score.attrs['error'] = score.attrs['error']
+    ceiled_score.attrs[Score.RAW_VALUES_KEY] = score
+    ceiled_score.attrs['ceiling'] = ceiling
+    return ceiled_score

brainscore_vision/benchmarks/baker2022/__init__.py

@@ -0,0 +1,9 @@
+from brainscore_vision import benchmark_registry
+from brainscore_vision.benchmarks.baker2022.benchmark import Baker2022AccuracyDeltaFrankenstein, \
+    Baker2022AccuracyDeltaFragmented, Baker2022InvertedAccuracyDelta
+
+DATASETS = ['normal', 'inverted']
+
+benchmark_registry['Baker2022frankenstein-accuracy_delta'] = lambda: Baker2022AccuracyDeltaFrankenstein()
+benchmark_registry['Baker2022fragmented-accuracy_delta'] = lambda: Baker2022AccuracyDeltaFragmented()
+benchmark_registry['Baker2022inverted-accuracy_delta'] = lambda: Baker2022InvertedAccuracyDelta()

brainscore_vision/benchmarks/baker2022/benchmark.py

@@ -0,0 +1,125 @@
+import numpy as np
+
+from brainscore_vision.benchmark_helpers import bound_score
+from brainscore_vision.benchmarks import BenchmarkBase
+from brainscore_vision.benchmark_helpers.screen import place_on_screen
+from brainscore_vision.metrics import Score
+from brainscore_vision.model_interface import BrainModel
+from brainscore_vision.utils import LazyLoad
+from typing import List
+from brainio.assemblies import DataAssembly
+from brainscore_vision import load_dataset, load_metric
+
+BIBTEX = """@article{BAKER2022104913,
+                title = {Deep learning models fail to capture the configural nature of human shape perception},
+                journal = {iScience},
+                volume = {25},
+                number = {9},
+                pages = {104913},
+                year = {2022},
+                issn = {2589-0042},
+                doi = {https://doi.org/10.1016/j.isci.2022.104913},
+                url = {https://www.sciencedirect.com/science/article/pii/S2589004222011853},
+                author = {Nicholas Baker and James H. Elder},
+                keywords = {Biological sciences, Neuroscience, Sensory neuroscience},
+                abstract = {Summary
+                A hallmark of human object perception is sensitivity to the holistic configuration of the local shape features of an object. Deep convolutional neural networks (DCNNs) are currently the dominant models for object recognition processing in the visual cortex, but do they capture this configural sensitivity? To answer this question, we employed a dataset of animal silhouettes and created a variant of this dataset that disrupts the configuration of each object while preserving local features. While human performance was impacted by this manipulation, DCNN performance was not, indicating insensitivity to object configuration. Modifications to training and architecture to make networks more brain-like did not lead to configural processing, and none of the networks were able to accurately predict trial-by-trial human object judgements. We speculate that to match human configural sensitivity, networks must be trained to solve a broader range of object tasks beyond category recognition.}
+        }"""
+
+DATASETS = ['normal', 'inverted']
+
+
+class _Baker2022AccuracyDelta(BenchmarkBase):
+    def __init__(self, identifier_suffix: str, dataset: str, image_types: List[str]):
+        """
+        :param identifier_suffix: how to uniquely identify this combination of dataset and image types
+        :param dataset: orientation of stimuli. Either 'normal' or 'inverted'
+        :param image_types: Either ["w", "f"] for frankenstein delta or ["w", "o"] for fragmented delta
+        """
+        self._metric = load_metric('baker_accuracy_delta', image_types=image_types)
+        self.image_types = image_types
+        self.orientation = dataset
+        self._ceiling = SplitHalvesConsistencyBaker(metric=self._metric,
+                                                    num_splits=100, split_coordinate="subject",
+                                                    image_types=self.image_types)
+        self._assembly = LazyLoad(lambda: load_assembly(dataset))
+        self._visual_degrees = 8.8
+        self._number_of_trials = 1
+
+        super(_Baker2022AccuracyDelta, self).__init__(
+            identifier=f'Baker2022{identifier_suffix}-accuracy_delta', version=1,
+            ceiling_func=lambda: self._ceiling(assembly=self._assembly),
+            parent='Baker2022',
+            bibtex=BIBTEX)
+
+    def __call__(self, candidate: BrainModel):
+        choice_labels = set(self._assembly['truth'].values)
+        choice_labels = list(sorted(choice_labels))
+        candidate.start_task(BrainModel.Task.label, choice_labels)
+        stimulus_set = place_on_screen(self._assembly.stimulus_set, target_visual_degrees=candidate.visual_degrees(),
+                                       source_visual_degrees=self._visual_degrees)
+        stimuli = stimulus_set[stimulus_set["orientation"] == self.orientation]
+        labels = candidate.look_at(stimuli, number_of_trials=self._number_of_trials)
+        assembly = self._assembly[self._assembly["orientation"] == self.orientation]
+        raw_score = self._metric(labels, assembly)
+        ceiling = self._ceiling(assembly)
+        score = raw_score / ceiling
+
+        bound_score(score)  # cap score at 1 if ceiled score > 1
+        score.attrs['raw'] = raw_score
+        score.attrs['ceiling'] = ceiling
+        return score
+
+
+def Baker2022AccuracyDeltaFrankenstein():
+    return _Baker2022AccuracyDelta(identifier_suffix='frankenstein', dataset='normal', image_types=["w", "f"])
+
+
+def Baker2022AccuracyDeltaFragmented():
+    return _Baker2022AccuracyDelta(identifier_suffix='fragmented', dataset='normal', image_types=["w", "o"])
+
+
+def Baker2022InvertedAccuracyDelta():
+    return _Baker2022AccuracyDelta(identifier_suffix='inverted', dataset='inverted', image_types=["w", "f"])
+
+
+def load_assembly(dataset):
+    assembly = load_dataset(f'Baker2022_{dataset}_distortion')
+    return assembly
+
+
+# ceiling method:
+class SplitHalvesConsistencyBaker:
+    def __init__(self, metric, num_splits: int, split_coordinate: str, image_types):
+        """
+        :param metric: the metric used to estimate a ceiling
+        :param num_splits: how many times to create two halves
+        :param split_coordinate: over which coordinate to split the assembly into halves
+        """
+        self._metric = metric
+        self.num_splits = num_splits
+        self.split_coordinate = split_coordinate
+        self.image_types = image_types
+
+    def __call__(self, assembly: DataAssembly) -> Score:
+        consistencies, uncorrected_consistencies = [], []
+        splits = range(self.num_splits)
+        random_state = np.random.RandomState(0)
+        for _ in splits:
+            num_subjects = len(set(assembly["subject"].values))
+            half1_subjects = random_state.choice(range(1, num_subjects), (num_subjects // 2), replace=False)
+            half1 = assembly[
+                {'presentation': [subject in half1_subjects for subject in assembly['subject'].values]}]
+            half2 = assembly[
+                {'presentation': [subject not in half1_subjects for subject in assembly['subject'].values]}]
+            consistency = self._metric.compute_ceiling(half1, half2, self.image_types)
+            uncorrected_consistencies.append(consistency)
+            # Spearman-Brown correction for sub-sampling
+            corrected_consistency = 2 * consistency / (1 + (2 - 1) * consistency)
+            consistencies.append(corrected_consistency)
+        consistencies = Score(consistencies, coords={'split': splits}, dims=['split'])
+        uncorrected_consistencies = Score(uncorrected_consistencies, coords={'split': splits}, dims=['split'])
+        average_consistency = consistencies.median('split')
+        average_consistency.attrs['raw'] = consistencies
+        average_consistency.attrs['uncorrected_consistencies'] = uncorrected_consistencies
+        return average_consistency

brainscore_vision/benchmarks/baker2022/requirements.txt

@@ -0,0 +1 @@
+numpy

brainscore_vision/benchmarks/baker2022/test.py

@@ -0,0 +1,90 @@
+from pathlib import Path
+import pytest
+from pytest import approx
+from brainio.assemblies import BehavioralAssembly
+from brainscore_vision import benchmark_registry, load_benchmark
+from brainscore_vision.benchmarks.baker2022 import DATASETS
+from brainscore_vision.benchmark_helpers import PrecomputedFeatures
+from brainscore_vision.data_helpers import s3
+
+
+@pytest.mark.private_access
+class TestBaker2022:
+
+    # ensure normal and inverted datasets are there
+    def test_count(self):
+        assert len(DATASETS) == 2
+
+    # ensure the three benchmarks themselves are there
+    @pytest.mark.parametrize('identifier', [
+        'Baker2022inverted-accuracy_delta',
+        'Baker2022fragmented-accuracy_delta',
+        'Baker2022frankenstein-accuracy_delta',
+    ])
+    def test_identifier(self, identifier):
+        assert identifier in benchmark_registry
+        benchmark = load_benchmark(identifier=identifier)
+        assert benchmark.identifier == identifier
+
+    # Test expected ceiling
+    @pytest.mark.parametrize('benchmark, expected_ceiling', [
+        ('Baker2022frankenstein-accuracy_delta', 0.8498),
+        ('Baker2022fragmented-accuracy_delta', 0.9385),
+        ('Baker2022inverted-accuracy_delta', 0.6538),
+    ])
+    def test_benchmark_ceiling(self, benchmark, expected_ceiling):
+        benchmark = load_benchmark(benchmark)
+        assembly = benchmark._assembly
+        if "inverted" in benchmark.identifier:
+            inverted_assembly = assembly[assembly["orientation"] == "inverted"]
+            ceiling = benchmark._ceiling(inverted_assembly)
+        else:
+            ceiling = benchmark._ceiling(assembly)
+        assert ceiling == approx(expected_ceiling, abs=0.001)
+
+    # Test raw scores
+    @pytest.mark.parametrize('benchmark, model, expected_raw_score', [
+        ('Baker2022frankenstein-accuracy_delta', 'resnet-50-pytorch', approx(0.2847, abs=0.0001)),
+        ('Baker2022fragmented-accuracy_delta', 'resnet-50-pytorch', approx(0.8452, abs=0.0001)),
+        ('Baker2022inverted-accuracy_delta', 'resnet-50-pytorch', approx(0.0, abs=0.0001)),
+        ('Baker2022frankenstein-accuracy_delta', 'resnet50-SIN', approx(0.6823, abs=0.0001)),
+        ('Baker2022fragmented-accuracy_delta', 'resnet50-SIN', approx(0.9100, abs=0.0001)),
+        ('Baker2022inverted-accuracy_delta', 'resnet50-SIN', approx(0.7050, abs=0.0001)),
+    ])
+    def test_model_raw_score(self, benchmark, model, expected_raw_score):
+
+        benchmark_object = load_benchmark(benchmark)
+        filename = f"{model}-{benchmark}.nc"
+        precomputed_features = Path(__file__).parent / filename
+        s3.download_file_if_not_exists(precomputed_features,
+                                       bucket='brainscore-vision',
+                                       remote_filepath=f'benchmarks/Baker2022/{filename}')
+        precomputed_features = BehavioralAssembly.from_files(file_path=precomputed_features)
+        precomputed_features = PrecomputedFeatures(precomputed_features, visual_degrees=8)
+        score = benchmark_object(precomputed_features)
+        raw_score = score.raw
+
+        # division by ceiling <= 1 should result in higher score
+        assert score >= raw_score
+        assert raw_score == expected_raw_score
+
+    # test ceiled score
+    @pytest.mark.parametrize('benchmark, model, expected_ceiled_score', [
+        ('Baker2022frankenstein-accuracy_delta', 'resnet-50-pytorch', approx(0.3350, abs=0.0001)),
+        ('Baker2022fragmented-accuracy_delta', 'resnet-50-pytorch', approx(0.9005, abs=0.0001)),
+        ('Baker2022inverted-accuracy_delta', 'resnet-50-pytorch', approx(0.0, abs=0.0001)),
+        ('Baker2022frankenstein-accuracy_delta', 'resnet50-SIN', approx(0.8029, abs=0.0001)),
+        ('Baker2022fragmented-accuracy_delta', 'resnet50-SIN', approx(0.9696, abs=0.0001)),
+        ('Baker2022inverted-accuracy_delta', 'resnet50-SIN', approx(1.000, abs=0.0001)),
+    ])
+    def test_model_ceiled_score(self, benchmark, model, expected_ceiled_score):
+        benchmark_object = load_benchmark(benchmark)
+        filename = f"{model}-{benchmark}.nc"
+        precomputed_features = Path(__file__).parent / filename
+        s3.download_file_if_not_exists(precomputed_features,
+                                       bucket='brainscore-vision',
+                                       remote_filepath=f'benchmarks/Baker2022/{filename}')
+        precomputed_features = BehavioralAssembly.from_files(file_path=precomputed_features)
+        precomputed_features = PrecomputedFeatures(precomputed_features, visual_degrees=8)
+        score = benchmark_object(precomputed_features)
+        assert score == expected_ceiled_score

brainscore_vision/benchmarks/bmd2024/__init__.py

@@ -0,0 +1,8 @@
+from brainscore_vision import benchmark_registry
+from brainscore_vision.benchmarks.bmd2024.benchmark import BMD2024AccuracyDistance
+
+# behavioral benchmarks
+benchmark_registry['BMD2024.texture_1Behavioral-accuracy_distance'] = lambda: BMD2024AccuracyDistance('texture_1')
+benchmark_registry['BMD2024.texture_2Behavioral-accuracy_distance'] = lambda: BMD2024AccuracyDistance('texture_2')
+benchmark_registry['BMD2024.dotted_1Behavioral-accuracy_distance'] = lambda: BMD2024AccuracyDistance('dotted_1')
+benchmark_registry['BMD2024.dotted_2Behavioral-accuracy_distance'] = lambda: BMD2024AccuracyDistance('dotted_2')

brainscore_vision/benchmarks/bmd2024/benchmark.py

@@ -0,0 +1,51 @@
+import numpy as np
+
+from brainio.assemblies import BehavioralAssembly
+from brainscore_vision import load_dataset, load_metric
+from brainscore_vision.benchmark_helpers.screen import place_on_screen
+from brainscore_vision.benchmarks import BenchmarkBase
+from brainscore_vision.metrics import Score
+from brainscore_vision.model_interface import BrainModel
+from brainscore_vision.utils import LazyLoad
+
+BIBTEX = ""  # to appear in a future article
+
+
+class _BMD_2024_BehavioralAccuracyDistance(BenchmarkBase):
+    # behavioral benchmark
+    def __init__(self, dataset):
+        self._visual_degrees = 8
+        self._metric = load_metric('accuracy_distance')
+        self._assembly = LazyLoad(lambda: load_assembly(dataset))
+        super(_BMD_2024_BehavioralAccuracyDistance, self).__init__(
+            identifier=f'BMD2024.{dataset}Behavioral-accuracy_distance', version=1,
+            ceiling_func=lambda: self._metric.ceiling(self._assembly),
+            parent='BMD2024',
+            bibtex=BIBTEX)
+
+    def __call__(self, candidate: BrainModel):
+        choice_labels = set(self._assembly.stimulus_set['truth'].values)
+        choice_labels = list(sorted(choice_labels))
+        candidate.start_task(BrainModel.Task.label, choice_labels)
+        stimulus_set = place_on_screen(self._assembly.stimulus_set,
+                                       target_visual_degrees=candidate.visual_degrees(),
+                                       source_visual_degrees=self._visual_degrees)
+        labels = candidate.look_at(stimulus_set, number_of_trials=1)
+        raw_score = self._metric(labels, self._assembly)
+        ceiling = self.ceiling
+        score = raw_score / ceiling
+        # ensure score <= 1.0
+        if score.values > 1:
+            score = Score(np.array(1.))
+        score.attrs['raw'] = raw_score
+        score.attrs['ceiling'] = ceiling
+        return score
+
+
+def load_assembly(dataset: str) -> BehavioralAssembly:
+    assembly = load_dataset(f'BMD2024.{dataset}')
+    return assembly
+
+
+def BMD2024AccuracyDistance(experiment):
+    return _BMD_2024_BehavioralAccuracyDistance(experiment)

brainscore_vision/benchmarks/bmd2024/test.py

@@ -0,0 +1,29 @@
+import pytest
+from pytest import approx
+
+from brainscore_vision import benchmark_registry, load_benchmark
+
+
+@pytest.mark.parametrize('benchmark', [
+    'BMD2024.texture_1Behavioral-accuracy_distance',
+    'BMD2024.texture_2Behavioral-accuracy_distance',
+    'BMD2024.dotted_1Behavioral-accuracy_distance',
+    'BMD2024.dotted_2Behavioral-accuracy_distance',
+])
+def test_benchmark_registry(benchmark):
+    assert benchmark in benchmark_registry
+
+
+class TestBehavioral:
+    @pytest.mark.private_access
+    @pytest.mark.parametrize('dataset, expected_ceiling', [
+        ('texture_1', approx(0.98283, abs=0.001)),
+        ('texture_2', approx(0.97337, abs=0.001)),
+        ('dotted_1', approx(0.97837, abs=0.001)),
+        ('dotted_2', approx(0.93071, abs=0.001)),  # all of the above are AccuracyDistance
+    ])
+    def test_dataset_ceiling(self, dataset, expected_ceiling):
+        benchmark = f"BMD2024.{dataset}Behavioral-accuracy_distance"
+        benchmark = load_benchmark(benchmark)
+        ceiling = benchmark.ceiling
+        assert ceiling == expected_ceiling

brainscore_vision/benchmarks/bracci2019/__init__.py

@@ -0,0 +1,8 @@
+from brainscore_vision import benchmark_registry
+from .benchmark import Bracci2019RSA
+
+# benchmark_registry['Bracci2019_RSA-V1'] = lambda: _Bracci2019RSA("V1")
+# benchmark_registry['Bracci2019_RSA-posteriorVTC'] = lambda: _Bracci2019RSA("posteriorVTC")
+benchmark_registry['Bracci2019.anteriorVTC-rdm'] = lambda: Bracci2019RSA("anteriorVTC")
+
+