brainscore-vision 2.1__py3-none-any.whl

brainscore_vision/benchmarks/ferguson2024/helpers/helpers.py

@@ -0,0 +1,251 @@
+import pandas as pd
+import numpy as np
+from brainio.assemblies import BehavioralAssembly
+import sympy as sp
+from pandas import DataFrame
+from tqdm import tqdm
+import statistics
+from typing import Dict
+
+# number of distractors in the experiment
+DISTRACTOR_NUMS = ["1.0", "5.0", "11.0"]
+
+# These are precomputed subject average lapse rates
+LAPSE_RATES = {'circle_line': 0.0335, 'color': 0.0578, 'convergence': 0.0372, 'eighth': 0.05556,
+               'gray_easy': 0.0414, 'gray_hard': 0.02305, 'half': 0.0637, 'juncture': 0.3715,
+               'lle': 0.0573, 'llh': 0.0402, 'quarter': 0.0534, 'round_f': 0.08196,
+               'round_v': 0.0561, 'tilted_line': 0.04986}
+
+# These are precomputed integral errors, computed by bootstrapping (see below)
+HUMAN_INTEGRAL_ERRORS = {'circle_line': 0.3078, 'color': 0.362, 'convergence': 0.2773, 'eighth': 0.278,
+                         'gray_easy': 0.309, 'gray_hard': 0.4246, 'half': 0.3661, 'juncture': 0.2198,
+                         'lle': 0.209, 'llh': 0.195, 'quarter': 0.2959, 'round_f': 0.344,
+                         'round_v': 0.2794, 'tilted_line': 0.3573}
+
+
+def get_adjusted_rate(acc: float, lapse_rate: float, n_way: int = 2) -> float:
+    """
+    - Adjusts the raw accuracy by a lapse rate correction
+
+    :param acc: float, the raw accuracy
+    :param lapse_rate: a precomputed float defined above that represents avg. subject lapse rate in experiment
+    :param n_way: int, (default value 2), the number of ways to divide by
+
+    :return: float, the SEM of that array
+    """
+    return (acc - lapse_rate * (1.0 / n_way)) / (1 - lapse_rate)
+
+
+def sem(array: BehavioralAssembly) -> float:
+    """
+    - Get the standard error of the mean (SEM) of an assembly
+
+    :param array: the assembly to look at
+    :return: float, the SEM of that array
+    """
+    array = np.array(array)
+    return np.std(array) / np.sqrt(len(array))
+
+
+def get_line(point_1: tuple, point_2: tuple) -> str:
+    """
+    - Calculate the equation of a line from two points
+
+    :param point_1: tuple in the form (x, y) of first point
+    :param point_2: tuple in the form (x, y) of second point
+    :return: str, equation of a line in the form y = mx + b
+    """
+    x1, y1 = point_1
+    x2, y2 = point_2
+    m = (y2 - y1) / (x2 - x1)
+    c = y1 - m * x1
+    equation = f"{m:.10f}*x + {c:.10f}"
+    return equation
+
+
+def integrate_line(equation: str, lower: float, upper: float) -> float:
+    """
+    - Integrates an equation.
+
+    :param equation: a string representing the equation of the line to integrate
+    :param lower: float, the lower bound of the definite integral
+    :param upper: float, the upper bound of the definite integral
+    :return: float, representing the integral of that line
+    """
+    x = sp.symbols('x')
+    integral_definite = sp.integrate(equation, (x, lower, upper))
+    return integral_definite
+
+
+def get_averages(df_blue: DataFrame, df_orange: DataFrame, num_distractors: str) -> (float, float):
+    """
+    - Gets the per-distractor averages for a block
+
+    :param df_blue: the first (blue) block of data (target on a field of distractors)
+    :param df_orange: the second (orange) block of data (distractor on a field of targets)
+    :param num_distractors:  string of a float representing how many distractors to look at
+    :return:
+    """
+    blue_df = df_blue[df_blue["distractor_nums"] == num_distractors]
+    orange_df = df_orange[df_orange["distractor_nums"] == num_distractors]
+    blue_val_avg = blue_df["correct"].values.mean()
+    orange_val_avg = orange_df["correct"].values.mean()
+    return blue_val_avg, orange_val_avg
+
+
+def calculate_integral(df_blue: DataFrame, df_orange: DataFrame) -> float:
+    """
+    - Manually calculates the integral under the delta line
+
+    :param df_blue: the first (blue) block of data (target on a field of distractors)
+    :param df_orange: the second (orange) block of data (distractor on a field of targets)
+    :return: float representing the integral of the delta line
+    """
+    blue_low_avg, orange_low_avg = get_averages(df_blue, df_orange, "1.0")
+    blue_mid_avg, orange_mid_avg = get_averages(df_blue, df_orange, "5.0")
+    blue_high_avg, orange_high_avg = get_averages(df_blue, df_orange, "11.0")
+
+    # compute deltas
+    low_delta = orange_low_avg - blue_low_avg
+    mid_delta = orange_mid_avg - blue_mid_avg
+    high_delta = orange_high_avg - blue_high_avg
+
+    # get equation of line through 1-5
+    point_1 = (1, low_delta)
+    point_2 = (5, mid_delta)
+    equation = get_line(point_1, point_2)
+    first_half = integrate_line(equation, 1, 5)
+
+    # get line 5-11 equation and integrate
+    point_3 = (11, high_delta)
+    equation_2 = get_line(point_2, point_3)
+    second_half = integrate_line(equation_2, 5, 11)
+
+    # add up integral
+    total_integral = round(first_half + second_half, 4)
+    return total_integral
+
+
+def calculate_accuracy(df: BehavioralAssembly, lapse_rate: float) -> float:
+    """
+    - Calculates a per-subject lapse rate-corrected accuracy for an assembly.
+    - Subject accuracy is averaged over all images with a certain distractor size and repetition coords (i.e. these
+      coords are mixed togather and the accuracy is calculated over this merged assembly).
+
+    :param df: DataFrame Object that contains experimental data
+    :param lapse_rate: a precomputed float defined above that represents avg. subject lapse rate in experiment
+    :return: float representing the adjusted (for lapse rate) accuracy of that subject
+    """
+    accuracy = len(df[df["correct"] == True]) / len(df)
+    adjusted_accuracy = get_adjusted_rate(accuracy, lapse_rate)
+    return adjusted_accuracy
+
+
+def generate_summary_df(assembly: BehavioralAssembly, lapse_rate: float, block: str) -> pd.DataFrame:
+    """
+
+    - Takes in raw assembly data and outputs a dataframe of summary statistics, used for benchmark.
+    - For each distractor size, accuracy is calculated per subject.
+
+    :param assembly: the data in the form of a BehavioralAssembly
+    :param lapse_rate: a precomputed float defined above that represents avg. subject lapse rate in experiment
+    :param block: str that defined what data to look at, "first (blue) or second (orange)
+    :return: a DataFrame object that contains needed summary data
+    """
+    filtered_data = assembly[(assembly["trial_type"] == "normal") & (assembly["block"] == block)]
+    participants = list(set(filtered_data['participant_id'].values))
+
+    summary_data = []
+    for subject in participants:
+        subject_data = filtered_data[filtered_data["participant_id"] == subject]
+        for distractor_num in DISTRACTOR_NUMS:
+            distractor_df = subject_data[subject_data["distractor_nums"] == str(distractor_num)]
+            if len(distractor_df) == 0:
+                continue
+            adjusted_acc = calculate_accuracy(distractor_df, lapse_rate)
+            summary_data.append({
+                'distractor_nums': distractor_num,
+                'participant_id': subject,
+                'correct': adjusted_acc
+            })
+    summary_df = pd.DataFrame(summary_data, columns=['distractor_nums', 'participant_id', 'correct'])
+    return summary_df
+
+
+def split_dataframe(df: BehavioralAssembly, seed: int) -> (BehavioralAssembly, BehavioralAssembly):
+    """
+    - Takes in one DF and splits it into two, randomly, on the presentation dim
+
+    :param df: The DataFrame (assembly) to split
+    :param seed: a seed for the numpy rng
+    :return: Two DataFrames (assemblies)
+    """
+    if seed is not None:
+        np.random.seed(seed)
+    shuffled_indices = np.random.permutation(df.presentation.size)
+    half = len(shuffled_indices) // 2
+    indices_1 = shuffled_indices[:half]
+    indices_2 = shuffled_indices[half:]
+    dataarray_1 = df.isel(presentation=indices_1)
+    dataarray_2 = df.isel(presentation=indices_2)
+    return dataarray_1, dataarray_2
+
+
+def get_acc_delta(df_blue: DataFrame, df_orange: DataFrame, num_dist: str) -> float:
+    """
+    Helper function for bootstrapping. Calculates an accuracy delta on a specific subject/distractor.
+
+    :param df_blue: DataFrame, the first (blue) block of data (target on a field of distractors)
+    :param df_orange: DataFrame, the second (orange) block of data (distractor on a field of targets)
+    :param num_dist: string, number of distractors
+    :return: float representing the requested accuracy delta.
+    """
+    d_blue = df_blue[df_blue["distractor_nums"] == num_dist]
+    d_orange = df_orange[df_orange["distractor_nums"] == num_dist]
+    sampled_blue = d_blue.sample(n=1, replace=True)
+    sampled_orange = d_orange.sample(n=1, replace=True)
+    accuracy_delta = sampled_blue["correct"].values[0] - sampled_orange["correct"].values[0]
+    return accuracy_delta
+
+
+def boostrap_integral(df_blue: DataFrame, df_orange: DataFrame, num_loops: int = 500) -> Dict:
+    """
+    Computes an error (std) on integral calculation by bootstrapping the integral via slices of subjects.
+
+    :param df_blue: DataFrame, the first (blue) block of data (target on a field of distractors)
+    :param df_orange: DataFrame, the second (orange) block of data (distractor on a field of targets)
+    :param num_loops: int, number of times the boostrap will run (and thus take the average)
+    :return: Dict of values {bootstrapped_integral, bootstrapped_integral_error)
+    """
+    num_subjects = len(set(df_blue["participant_id"]))
+    integral_list = []
+    for i in tqdm(range(num_loops)):
+        accuracy_delta_lows = []
+        accuracy_delta_mids = []
+        accuracy_delta_highs = []
+        for j in range(num_subjects):
+            accuracy_delta_lows.append(get_acc_delta(df_blue, df_orange, num_dist="1.0"))  # get low distractor case
+            accuracy_delta_mids.append(get_acc_delta(df_blue, df_orange, num_dist="5.0"))  # get mid distractor case
+            accuracy_delta_highs.append(get_acc_delta(df_blue, df_orange, num_dist="11.0"))  # get high distractor case
+        average_low_delta = statistics.mean(accuracy_delta_highs)
+        average_mid_delta = statistics.mean(accuracy_delta_mids)
+        average_high_delta = statistics.mean(accuracy_delta_lows)
+
+        # get equation for line through points 1 - 5 and integrate:
+        point_1 = (1, average_low_delta)
+        point_2 = (5, average_mid_delta)
+        equation = get_line(point_1, point_2)
+        first_half = integrate_line(equation, 1, 5)
+
+        # get line 5-11 equation and integrate
+        point_3 = (11, average_high_delta)
+        equation_2 = get_line(point_2, point_3)
+        second_half = integrate_line(equation_2, 5, 11)
+
+        total_integral = first_half + second_half
+        integral_list.append(total_integral)
+    data_array = np.array(integral_list, dtype=float)
+    integral_mean = -np.mean(data_array)
+    integral_std = np.std(data_array)
+
+    return {"bootstrap_integral_mean": integral_mean, "integral_std": integral_std}

brainscore_vision/benchmarks/ferguson2024/requirements.txt

@@ -0,0 +1,5 @@
+pandas
+sympy
+tqdm
+statistics
+xarray

brainscore_vision/benchmarks/ferguson2024/test.py

@@ -0,0 +1,114 @@
+from pathlib import Path
+import pytest
+from pytest import approx
+from brainio.assemblies import BehavioralAssembly
+from brainscore_vision import load_benchmark
+from brainscore_vision.benchmark_helpers import PrecomputedFeatures
+from brainscore_vision.data_helpers import s3
+
+
+class TestExist:
+    @pytest.mark.parametrize("identifier", [
+        'Ferguson2024circle_line-value_delta',
+        'Ferguson2024color-value_delta',
+        'Ferguson2024convergence-value_delta',
+        'Ferguson2024eighth-value_delta',
+        'Ferguson2024gray_easy-value_delta',
+        'Ferguson2024gray_hard-value_delta',
+        'Ferguson2024half-value_delta',
+        'Ferguson2024juncture-value_delta',
+        'Ferguson2024lle-value_delta',
+        'Ferguson2024llh-value_delta',
+        'Ferguson2024quarter-value_delta',
+        'Ferguson2024round_f-value_delta',
+        'Ferguson2024round_v-value_delta',
+        'Ferguson2024tilted_line-value_delta',
+    ])
+    def test_benchmark_registry(self, identifier):
+        benchmark = load_benchmark(identifier)
+        assert benchmark is not None
+        assert benchmark.identifier == identifier
+
+
+class TestBehavioral:
+    @pytest.mark.private_access
+    @pytest.mark.parametrize('benchmark, expected_ceiling', [
+        ('Ferguson2024circle_line-value_delta', approx(0.883, abs=0.001)),
+        ('Ferguson2024color-value_delta', approx(0.897, abs=0.001)),
+        ('Ferguson2024convergence-value_delta', approx(0.862, abs=0.001)),
+        ('Ferguson2024eighth-value_delta', approx(0.852, abs=0.001)),
+        ('Ferguson2024gray_easy-value_delta', approx(0.907, abs=0.001)),
+        ('Ferguson2024gray_hard-value_delta', approx(0.863, abs=0.001)),
+        ('Ferguson2024half-value_delta', approx(0.898, abs=0.001)),
+        ('Ferguson2024juncture-value_delta', approx(0.767, abs=0.001)),
+        ('Ferguson2024lle-value_delta', approx(0.831, abs=0.001)),
+        ('Ferguson2024llh-value_delta', approx(0.812, abs=0.001)),
+        ('Ferguson2024quarter-value_delta', approx(0.876, abs=0.001)),
+        ('Ferguson2024round_f-value_delta', approx(0.874, abs=0.001)),
+        ('Ferguson2024round_v-value_delta', approx(0.853, abs=0.001)),
+        ('Ferguson2024tilted_line-value_delta', approx(0.912, abs=0.001)),
+    ])
+    def test_benchmark_ceiling(self, benchmark, expected_ceiling):
+        benchmark = load_benchmark(benchmark)
+        ceiling = benchmark._ceiling
+        assert ceiling == expected_ceiling
+
+    @pytest.mark.private_access
+    @pytest.mark.parametrize('benchmark, expected_raw_score', [
+        ('Ferguson2024circle_line-value_delta', approx(0.143, abs=0.001)),
+        ('Ferguson2024color-value_delta', approx(0.645, abs=0.001)),
+        ('Ferguson2024convergence-value_delta', approx(0.024, abs=0.001)),
+        ('Ferguson2024eighth-value_delta', approx(0.093, abs=0.001)),
+        ('Ferguson2024gray_easy-value_delta', approx(0.799, abs=0.001)),
+        ('Ferguson2024gray_hard-value_delta', approx(0.609, abs=0.001)),
+        ('Ferguson2024half-value_delta', approx(0.379, abs=0.001)),
+        ('Ferguson2024juncture-value_delta', approx(0.191, abs=0.001)),
+        ('Ferguson2024lle-value_delta', approx(0.208, abs=0.001)),
+        ('Ferguson2024llh-value_delta', approx(0.654, abs=0.001)),
+        ('Ferguson2024quarter-value_delta', approx(0.223, abs=0.001)),
+        ('Ferguson2024round_f-value_delta', approx(0.455, abs=0.001)),
+        ('Ferguson2024round_v-value_delta', approx(0.212, abs=0.001)),
+        ('Ferguson2024tilted_line-value_delta', approx(0.445, abs=0.001)),
+    ])
+    def test_model_raw_score(self, benchmark, expected_raw_score):
+        benchmark_object = load_benchmark(benchmark)
+        filename = f"alexnet_{benchmark}.nc"
+        precomputed_features = Path(__file__).parent / filename
+        s3.download_file_if_not_exists(precomputed_features,
+                                       bucket='brainscore-vision', remote_filepath=f'benchmarks/Ferguson2024/{filename}')
+        precomputed_features = BehavioralAssembly.from_files(file_path=precomputed_features)
+        precomputed_features = PrecomputedFeatures(precomputed_features, visual_degrees=8)
+        score = benchmark_object(precomputed_features)
+        raw_score = score.raw
+        # division by ceiling <= 1 should result in higher score
+        assert score >= raw_score
+        assert raw_score == expected_raw_score
+
+    @pytest.mark.private_access
+    @pytest.mark.parametrize('benchmark, expected_ceiled_score', [
+        ('Ferguson2024circle_line-value_delta', approx(0.162, abs=0.001)),
+        ('Ferguson2024color-value_delta', approx(0.719, abs=0.001)),
+        ('Ferguson2024convergence-value_delta', approx(0.028, abs=0.001)),
+        ('Ferguson2024eighth-value_delta', approx(0.109, abs=0.001)),
+        ('Ferguson2024gray_easy-value_delta', approx(0.882, abs=0.001)),
+        ('Ferguson2024gray_hard-value_delta', approx(0.706, abs=0.001)),
+        ('Ferguson2024half-value_delta', approx(0.423, abs=0.001)),
+        ('Ferguson2024juncture-value_delta', approx(0.248, abs=0.001)),
+        ('Ferguson2024lle-value_delta', approx(0.250, abs=0.001)),
+        ('Ferguson2024llh-value_delta', approx(0.805, abs=0.001)),
+        ('Ferguson2024quarter-value_delta', approx(0.255, abs=0.001)),
+        ('Ferguson2024round_f-value_delta', approx(0.520, abs=0.001)),
+        ('Ferguson2024round_v-value_delta', approx(0.249, abs=0.001)),
+        ('Ferguson2024tilted_line-value_delta', approx(0.489, abs=0.001)),
+    ])
+    def test_model_ceiled_score(self, benchmark, expected_ceiled_score):
+        benchmark_object = load_benchmark(benchmark)
+        filename = f"alexnet_{benchmark}.nc"
+        precomputed_features = Path(__file__).parent / filename
+        s3.download_file_if_not_exists(precomputed_features,
+                                       bucket='brainscore-vision', remote_filepath=f'benchmarks/Ferguson2024/{filename}')
+        precomputed_features = BehavioralAssembly.from_files(file_path=precomputed_features)
+        precomputed_features = PrecomputedFeatures(precomputed_features, visual_degrees=8)
+        score = benchmark_object(precomputed_features)
+        assert score == expected_ceiled_score
+

brainscore_vision/benchmarks/freemanziemba2013/__init__.py

@@ -0,0 +1,10 @@
+from brainscore_vision import benchmark_registry
+from .benchmarks.benchmark import MovshonFreemanZiemba2013V1PLS, MovshonFreemanZiemba2013V2PLS
+from .benchmarks.public_benchmarks import FreemanZiembaV1PublicBenchmark, FreemanZiembaV2PublicBenchmark
+
+benchmark_registry['FreemanZiemba2013.V1-pls'] = MovshonFreemanZiemba2013V1PLS
+benchmark_registry['FreemanZiemba2013.V2-pls'] = MovshonFreemanZiemba2013V2PLS
+
+# public benchmarks
+benchmark_registry['FreemanZiemba2013public.V1-pls'] = FreemanZiembaV1PublicBenchmark
+benchmark_registry['FreemanZiemba2013public.V2-pls'] = FreemanZiembaV2PublicBenchmark

brainscore_vision/benchmarks/freemanziemba2013/benchmarks/benchmark.py

@@ -0,0 +1,53 @@
+from result_caching import store
+
+from brainscore_vision import load_dataset, load_metric, load_ceiling
+from brainscore_vision.benchmark_helpers.neural_common import NeuralBenchmark, average_repetition
+from brainscore_vision.data.freemanziemba2013 import BIBTEX
+from brainscore_vision.utils import LazyLoad
+
+VISUAL_DEGREES = 4
+NUMBER_OF_TRIALS = 20
+
+
+def _MovshonFreemanZiemba2013Region(region, identifier_metric_suffix, similarity_metric, ceiler):
+    assembly_repetition = LazyLoad(lambda region=region: load_assembly(False, region=region))
+    assembly = LazyLoad(lambda region=region: load_assembly(True, region=region))
+    return NeuralBenchmark(identifier=f'FreemanZiemba2013.{region}-{identifier_metric_suffix}', version=2,
+                           assembly=assembly, similarity_metric=similarity_metric, parent=region,
+                           ceiling_func=lambda: ceiler(assembly_repetition),
+                           visual_degrees=VISUAL_DEGREES, number_of_trials=NUMBER_OF_TRIALS,
+                           bibtex=BIBTEX)
+
+
+def MovshonFreemanZiemba2013V1PLS():
+    metric = load_metric('pls', crossvalidation_kwargs=dict(stratification_coord='texture_type'))
+    ceiler = load_ceiling('internal_consistency')
+    return _MovshonFreemanZiemba2013Region(
+        'V1', identifier_metric_suffix='pls', similarity_metric=metric, ceiler=ceiler)
+
+
+def MovshonFreemanZiemba2013V2PLS():
+    metric = load_metric('pls', crossvalidation_kwargs=dict(stratification_coord='texture_type'))
+    ceiler = load_ceiling('internal_consistency')
+    return _MovshonFreemanZiemba2013Region(
+        'V2', identifier_metric_suffix='pls', similarity_metric=metric, ceiler=ceiler)
+
+
+@store()
+def load_assembly(average_repetitions, region, access='private'):
+    assembly = load_dataset(f'FreemanZiemba2013.{access}')
+    assembly = assembly.sel(region=region)
+    assembly = assembly.stack(neuroid=['neuroid_id'])  # work around xarray multiindex issues
+    assembly['region'] = 'neuroid', [region] * len(assembly['neuroid'])
+    assembly.load()
+    time_window = (50, 200)
+    assembly = assembly.sel(time_bin=[(t, t + 1) for t in range(*time_window)])
+    assembly = assembly.mean(dim='time_bin', keep_attrs=True)
+    assembly = assembly.expand_dims('time_bin_start').expand_dims('time_bin_end')
+    assembly['time_bin_start'], assembly['time_bin_end'] = [time_window[0]], [time_window[1]]
+    assembly = assembly.stack(time_bin=['time_bin_start', 'time_bin_end'])
+    assembly = assembly.squeeze('time_bin')
+    assembly = assembly.transpose('presentation', 'neuroid')
+    if average_repetitions:
+        assembly = average_repetition(assembly)
+    return assembly

brainscore_vision/benchmarks/freemanziemba2013/benchmarks/public_benchmarks.py

@@ -0,0 +1,37 @@
+"""
+The purpose of this file is to provide benchmarks based on publicly accessible data that can be run on candidate models
+without restrictions. As opposed to the private benchmarks hosted on www.Brain-Score.org, models can be evaluated
+without having to submit them to the online platform.
+This allows for quick local prototyping, layer commitment, etc.
+For the final model evaluation, candidate models should still be sent to www.Brain-Score.org to evaluate them on
+held-out private data.
+"""
+import logging
+
+from brainscore_vision import load_metric, load_ceiling
+from brainscore_vision.benchmark_helpers.neural_common import NeuralBenchmark
+from brainscore_vision.utils import LazyLoad
+from .benchmark import load_assembly, NUMBER_OF_TRIALS, VISUAL_DEGREES, BIBTEX
+
+_logger = logging.getLogger(__name__)
+
+
+def _freemanziemba2013_public_benchmark(region: str):
+    assembly_repetition = LazyLoad(lambda: load_assembly(region=region, access='public', average_repetitions=False))
+    assembly = LazyLoad(lambda: load_assembly(region=region, access='public', average_repetitions=True))
+    similarity_metric = load_metric('pls', crossvalidation_kwargs=dict(stratification_coord='texture_type'))
+    ceiler = load_ceiling('internal_consistency')
+    return NeuralBenchmark(identifier=f"FreemanZiemba2013.{region}.public-pls", version=1,
+                           assembly=assembly, similarity_metric=similarity_metric,
+                           visual_degrees=VISUAL_DEGREES, number_of_trials=NUMBER_OF_TRIALS,
+                           ceiling_func=lambda: ceiler(assembly_repetition),
+                           parent=None,
+                           bibtex=BIBTEX)
+
+
+def FreemanZiembaV1PublicBenchmark():
+    return _freemanziemba2013_public_benchmark('V1')
+
+
+def FreemanZiembaV2PublicBenchmark():
+    return _freemanziemba2013_public_benchmark('V2')

brainscore_vision/benchmarks/freemanziemba2013/test.py

@@ -0,0 +1,98 @@
+from pathlib import Path
+
+import pytest
+from pytest import approx
+
+from brainscore_vision import benchmark_registry, load_benchmark
+from brainscore_vision.benchmark_helpers import PrecomputedFeatures
+from brainscore_vision.benchmark_helpers.test_helper import StandardizedTests, PrecomputedTests, NumberOfTrialsTests, \
+    VisualDegreesTests
+from brainscore_vision.data_helpers import s3
+
+standardized_tests = StandardizedTests()
+precomputed_test = PrecomputedTests()
+num_trials_test = NumberOfTrialsTests()
+visual_degrees_test = VisualDegreesTests()
+
+
+@pytest.mark.parametrize('benchmark', [
+    'FreemanZiemba2013.V1-pls',
+    'FreemanZiemba2013.V2-pls',
+])
+def test_benchmark_registry(benchmark):
+    assert benchmark in benchmark_registry
+
+
+@pytest.mark.memory_intense
+@pytest.mark.parametrize('benchmark, expected', [
+    ('FreemanZiemba2013.V1-pls', approx(.873345, abs=.001)),
+    ('FreemanZiemba2013.V2-pls', approx(.824836, abs=.001)),
+])
+def test_ceilings(benchmark, expected):
+    standardized_tests.ceilings_test(benchmark, expected)
+
+
+@pytest.mark.memory_intense
+@pytest.mark.parametrize('benchmark, visual_degrees, expected', [
+    ('FreemanZiemba2013.V1-pls', 4, approx(.668491, abs=.001)),
+    ('FreemanZiemba2013.V2-pls', 4, approx(.553155, abs=.001)),
+])
+def test_self_regression(benchmark, visual_degrees, expected):
+    standardized_tests.self_regression_test(benchmark, visual_degrees, expected)
+
+
+@pytest.mark.memory_intense
+@pytest.mark.parametrize('benchmark, expected', [
+    ('FreemanZiemba2013.V1-pls', approx(.466222, abs=.005)),
+    ('FreemanZiemba2013.V2-pls', approx(.459283, abs=.005)),
+])
+def test_FreemanZiemba2013(benchmark, expected):
+    filename = 'alexnet-freemanziemba2013.aperture-private.nc'
+    filepath = Path(__file__).parent / filename
+    s3.download_file_if_not_exists(local_path=filepath,
+                                   bucket='brain-score-tests', remote_filepath=f'tests/test_benchmarks/{filename}')
+    precomputed_test.run_test(benchmark=benchmark, precomputed_features_filepath=filepath, expected=expected)
+
+
+@pytest.mark.parametrize('benchmark, candidate_degrees, image_id, expected', [
+    pytest.param('FreemanZiemba2013.V1-pls', 14, 'c3a633a13e736394f213ddf44bf124fe80cabe07',
+                 approx(.31429, abs=.0001), marks=[pytest.mark.private_access]),
+    pytest.param('FreemanZiemba2013.V1-pls', 6, 'c3a633a13e736394f213ddf44bf124fe80cabe07',
+                 approx(.22966, abs=.0001), marks=[pytest.mark.private_access]),
+    pytest.param('FreemanZiemba2013public.V1-pls', 14, '21041db1f26c142812a66277c2957fb3e2070916',
+                 approx(.314561, abs=.0001), marks=[]),
+    pytest.param('FreemanZiemba2013public.V1-pls', 6, '21041db1f26c142812a66277c2957fb3e2070916',
+                 approx(.23113, abs=.0001), marks=[]),
+    pytest.param('FreemanZiemba2013.V2-pls', 14, 'c3a633a13e736394f213ddf44bf124fe80cabe07',
+                 approx(.31429, abs=.0001), marks=[pytest.mark.private_access]),
+    pytest.param('FreemanZiemba2013.V2-pls', 6, 'c3a633a13e736394f213ddf44bf124fe80cabe07',
+                 approx(.22966, abs=.0001), marks=[pytest.mark.private_access]),
+    pytest.param('FreemanZiemba2013public.V2-pls', 14, '21041db1f26c142812a66277c2957fb3e2070916',
+                 approx(.314561, abs=.0001), marks=[]),
+    pytest.param('FreemanZiemba2013public.V2-pls', 6, '21041db1f26c142812a66277c2957fb3e2070916',
+                 approx(.23113, abs=.0001), marks=[]),
+])
+def test_amount_gray(benchmark: str, candidate_degrees: int, image_id: str, expected: float):
+    visual_degrees_test.amount_gray_test(benchmark, candidate_degrees, image_id, expected)
+
+
+@pytest.mark.private_access
+@pytest.mark.parametrize('benchmark_identifier', [
+    'FreemanZiemba2013.V1-pls',
+    'FreemanZiemba2013.V2-pls',
+])
+def test_repetitions(benchmark_identifier):
+    num_trials_test.repetitions_test(benchmark_identifier)
+
+
+@pytest.mark.memory_intense
+@pytest.mark.parametrize('benchmark, visual_degrees, expected', [
+    ('FreemanZiemba2013public.V1-pls', 4, approx(.679954, abs=.001)),
+    ('FreemanZiemba2013public.V2-pls', 4, approx(.577498, abs=.001)),
+])
+def test_self(benchmark, visual_degrees, expected):
+    benchmark = load_benchmark(benchmark)
+    source = benchmark._assembly.copy()
+    source = {benchmark._assembly.stimulus_set.identifier: source}
+    score = benchmark(PrecomputedFeatures(source, visual_degrees=visual_degrees)).raw
+    assert score == expected

brainscore_vision/benchmarks/geirhos2021/__init__.py

@@ -0,0 +1,59 @@
+from brainscore_vision import benchmark_registry
+from . import benchmark
+
+DATASETS = ['colour', 'contrast', 'cue-conflict', 'edge',
+            'eidolonI', 'eidolonII', 'eidolonIII',
+            'false-colour', 'high-pass', 'low-pass', 'phase-scrambling', 'power-equalisation',
+            'rotation', 'silhouette', 'sketch', 'stylized', 'uniform-noise']
+
+benchmark_registry['Geirhos2021colour-top1'] = getattr(benchmark, "Geirhos2021colourAccuracy")
+benchmark_registry['Geirhos2021colour-error_consistency'] = getattr(benchmark, "Geirhos2021colourErrorConsistency")
+
+benchmark_registry['Geirhos2021contrast-top1'] = getattr(benchmark, "Geirhos2021contrastAccuracy")
+benchmark_registry['Geirhos2021contrast-error_consistency'] = getattr(benchmark, "Geirhos2021contrastErrorConsistency")
+
+benchmark_registry['Geirhos2021cueconflict-top1'] = getattr(benchmark, "Geirhos2021cueconflictAccuracy")
+benchmark_registry['Geirhos2021cueconflict-error_consistency'] = getattr(benchmark, "Geirhos2021cueconflictErrorConsistency")
+
+benchmark_registry['Geirhos2021edge-top1'] = getattr(benchmark, "Geirhos2021edgeAccuracy")
+benchmark_registry['Geirhos2021edge-error_consistency'] = getattr(benchmark, "Geirhos2021edgeErrorConsistency")
+
+benchmark_registry['Geirhos2021eidolonI-top1'] = getattr(benchmark, "Geirhos2021eidolonIAccuracy")
+benchmark_registry['Geirhos2021eidolonI-error_consistency'] = getattr(benchmark, "Geirhos2021eidolonIErrorConsistency")
+
+benchmark_registry['Geirhos2021eidolonII-top1'] = getattr(benchmark, "Geirhos2021eidolonIIAccuracy")
+benchmark_registry['Geirhos2021eidolonII-error_consistency'] = getattr(benchmark, "Geirhos2021eidolonIIErrorConsistency")
+
+benchmark_registry['Geirhos2021eidolonIII-top1'] = getattr(benchmark, "Geirhos2021eidolonIIIAccuracy")
+benchmark_registry['Geirhos2021eidolonIII-error_consistency'] = getattr(benchmark, "Geirhos2021eidolonIIIErrorConsistency")
+
+benchmark_registry['Geirhos2021falsecolour-top1'] = getattr(benchmark, "Geirhos2021falsecolourAccuracy")
+benchmark_registry['Geirhos2021falsecolour-error_consistency'] = getattr(benchmark, "Geirhos2021falsecolourErrorConsistency")
+
+benchmark_registry['Geirhos2021highpass-top1'] = getattr(benchmark, "Geirhos2021highpassAccuracy")
+benchmark_registry['Geirhos2021highpass-error_consistency'] = getattr(benchmark, "Geirhos2021highpassErrorConsistency")
+
+benchmark_registry['Geirhos2021lowpass-top1'] = getattr(benchmark, "Geirhos2021lowpassAccuracy")
+benchmark_registry['Geirhos2021lowpass-error_consistency'] = getattr(benchmark, "Geirhos2021lowpassErrorConsistency")
+
+benchmark_registry['Geirhos2021phasescrambling-top1'] = getattr(benchmark, "Geirhos2021phasescramblingAccuracy")
+benchmark_registry['Geirhos2021phasescrambling-error_consistency'] = getattr(benchmark, "Geirhos2021phasescramblingErrorConsistency")
+
+benchmark_registry['Geirhos2021powerequalisation-top1'] = getattr(benchmark, "Geirhos2021powerequalisationAccuracy")
+benchmark_registry['Geirhos2021powerequalisation-error_consistency'] = getattr(benchmark, "Geirhos2021powerequalisationErrorConsistency")
+
+benchmark_registry['Geirhos2021rotation-top1'] = getattr(benchmark, "Geirhos2021rotationAccuracy")
+benchmark_registry['Geirhos2021rotation-error_consistency'] = getattr(benchmark, "Geirhos2021rotationErrorConsistency")
+
+benchmark_registry['Geirhos2021silhouette-top1'] = getattr(benchmark, "Geirhos2021silhouetteAccuracy")
+benchmark_registry['Geirhos2021silhouette-error_consistency'] = getattr(benchmark, "Geirhos2021silhouetteErrorConsistency")
+
+benchmark_registry['Geirhos2021sketch-top1'] = getattr(benchmark, "Geirhos2021sketchAccuracy")
+benchmark_registry['Geirhos2021sketch-error_consistency'] = getattr(benchmark, "Geirhos2021sketchErrorConsistency")
+
+benchmark_registry['Geirhos2021stylized-top1'] = getattr(benchmark, "Geirhos2021stylizedAccuracy")
+benchmark_registry['Geirhos2021stylized-error_consistency'] = getattr(benchmark, "Geirhos2021stylizedErrorConsistency")
+
+benchmark_registry['Geirhos2021uniformnoise-top1'] = getattr(benchmark, "Geirhos2021uniformnoiseAccuracy")
+benchmark_registry['Geirhos2021uniformnoise-error_consistency'] = getattr(benchmark, "Geirhos2021uniformnoiseErrorConsistency")
+