brainscore-vision 2.1__py3-none-any.whl

brainscore_vision/benchmarks/igustibagus2024/domain_transfer_analysis.py

@@ -0,0 +1,306 @@
+# import general libraries
+import copy
+
+import numpy as np
+import pandas as pd
+from sklearn.linear_model import RidgeClassifierCV
+from sklearn.model_selection import train_test_split
+from sklearn.pipeline import Pipeline
+from sklearn.preprocessing import StandardScaler
+from tqdm import tqdm
+
+# import brain-score specific libraries
+from brainscore_core import Score
+from brainscore_vision import BrainModel, load_stimulus_set
+from brainscore_vision.benchmark_helpers.screen import place_on_screen
+from brainscore_vision.benchmarks import BenchmarkBase
+
+#### define constants ####
+VISUAL_DEGREES = 8
+NUMBER_OF_TRIALS = 63
+CATEGORIES = ['apple', 'bear', 'bird', 'car', 'chair', 'dog', 'elephant', 'face', 'plane', 'zebra']
+SILHOUETTE_DOMAINS = ['convex_hull', 'outline', 'skeleton', 'silhouette']
+HVM_TEST_IMAGES_NUM = 30
+OOD_TEST_IMAGES_NUM = 30
+NUM_SPLITS = 1000
+
+BIBTEX = """
+@inproceedings{bagus2022primate,
+  title={Primate inferotemporal cortex neurons generalize better to novel image distributions than analogous deep neural networks units},
+  author={Bagus, Ayu Marliawaty I Gusti and Marques, Tiago and Sanghavi, Sachi and DiCarlo, James J and Schrimpf, Martin},
+  booktitle={SVRHM 2022 Workshop@ NeurIPS},
+  year={2022}
+}
+"""
+
+
+#### Analysis Benchmark Implementation ####
+
+class _OOD_AnalysisBenchmark(BenchmarkBase):
+    def __init__(self, classifier):
+        self._classifier = classifier
+        self._stimuli = load_stimulus_set('Igustibagus2024')
+        self._stimuli.identifier = 'domain_transfer_pico_oleo'
+        self._visual_degrees = 8
+        super(_OOD_AnalysisBenchmark, self).__init__(
+            identifier='Igustibagus2024-IT_accuracy',
+            version=1,
+            ceiling_func=lambda: Score([1, np.nan], coords={'aggregation': ['center', 'error']}, dims=['aggregation']),
+            parent='engineering',
+            bibtex=BIBTEX,
+        )
+
+    # The __call__ method takes as input a candidate BrainModel and outputs a similarity score of how brain-like
+    # the candidate is under this benchmark.
+    # A candidate here could be a model such as CORnet or brain-mapped Alexnet, but importantly the benchmark can be
+    # agnostic to the details of the candidate and instead only engage with the BrainModel interface.
+    def __call__(self, candidate: BrainModel) -> Score:
+        # based on the visual degrees of the candidate
+        fitting_stimuli = place_on_screen(self._stimuli, target_visual_degrees=candidate.visual_degrees(),
+                                          source_visual_degrees=self._visual_degrees)
+        candidate.start_recording('IT', time_bins=[(70, 170)])
+        activations = candidate.look_at(fitting_stimuli)
+        activations = activations.squeeze()
+        activations = activations.transpose('presentation', 'neuroid')
+
+        crossdomain_data_dict = get_crossdomain_data_dictionary(domain_transfer_data=activations)
+
+        crossdomain_results = []
+        for split in tqdm(np.arange(NUM_SPLITS)):
+            crossdomain_test_images_dict, complete_training_data = split_training_test_images(
+                crossdomain_data_dictionary=crossdomain_data_dict)
+
+            # Train the decoder #
+            decoder = get_decoder(data=complete_training_data, estimator=self._classifier)
+            for crossdomain in crossdomain_test_images_dict.keys():
+                test_accuracy = get_classifier_score_2AFC(classifier=decoder,
+                                                          data=crossdomain_test_images_dict[crossdomain])
+                test_accuracy = Score(test_accuracy)
+                test_accuracy = test_accuracy.expand_dims('domain').expand_dims('split')
+                test_accuracy['domain'] = [crossdomain]
+                test_accuracy['split'] = [split]
+                crossdomain_results.append(test_accuracy)
+        crossdomain_results = Score.merge(*crossdomain_results)
+
+        center = crossdomain_results.mean(dim='split').mean(dim='domain')
+        error = crossdomain_results.mean(dim='split').std(dim='domain')
+        score = Score([center, error], coords={'aggregation': ['center', 'error']}, dims=('aggregation',))
+        score.attrs[Score.RAW_VALUES_KEY] = crossdomain_results
+        return score
+
+
+def OOD_AnalysisBenchmark():
+    return _OOD_AnalysisBenchmark(
+        classifier=Pipeline([
+            ('scaler', StandardScaler()), 
+            ('classifier', RidgeClassifierCV(alphas=[0.0001, 0.001, 0.01, 0.1, 1, 10], fit_intercept=True))
+        ])
+    )
+
+
+########## helpers #############
+def get_crossdomain_data_dictionary(domain_transfer_data):
+    '''
+    Create a dictionary with each crossdomain data as key and its data as values
+
+    Arguments:
+        domain_transfer_data (NeuronRecordingAssembly): complete dataset
+
+    Returns:
+          dictionary with each crossdomain data as key and its data as values
+    '''
+    # Create dictionary
+    crossdomain_data_dict = {}
+    crossdomains = ['original', 'cartoon', 'line_drawing', 'mosaic', 'painting', 'sketch', 'convex_hull', 'outline',
+                    'skeleton', 'silhouette', 'cococolor', 'cocogray', 'tdw']
+    crossdomain_image_source = ['Silhouette', 'Art', 'Art', 'Art', 'Art', 'Art', 'Silhouette', 'Silhouette',
+                                'Silhouette', 'Silhouette', 'COCOColor', 'COCOGray', 'TDW']
+    for image_source, object_style in zip(crossdomain_image_source, crossdomains):
+        crossdomain_data = get_single_domain_data(data=domain_transfer_data, image_source_in_domain=image_source,
+                                                  object_style_in_domain=object_style)
+        if object_style == 'original':
+            crossdomain_data_dict['hvm'] = crossdomain_data
+        else:
+            crossdomain_data_dict[object_style] = crossdomain_data
+
+    return crossdomain_data_dict
+
+
+def get_single_domain_data(data, image_source_in_domain, object_style_in_domain):
+    '''
+    Filters the data for a single domain
+
+    Arguments:
+        data: NeuronRecordingAssembly xarray
+        image_source_in_domain (str): Image source of wanted domain
+        object_style_in_domain (str): Image style of wanted domain
+
+    Returns:
+        domain_data: data for single domain
+    '''
+    # Get domain data
+    if image_source_in_domain in ['Art', 'Silhouette']:
+        try:
+            domain_data = data.where(
+                (data.identifier == image_source_in_domain) & (data.object_style == object_style_in_domain), drop=True)
+        except:
+            domain_data = data.where(
+                (data.stimulus_source == image_source_in_domain) & (data.object_style == object_style_in_domain),
+                drop=True)
+
+    else:
+        try:
+            domain_data = data.where(data.identifier == image_source_in_domain, drop=True)
+        except:
+            domain_data = data.where(data.stimulus_source == image_source_in_domain, drop=True)
+
+    return domain_data
+
+
+def get_crossdomain_dataframes(single_neuron_image=False):
+    '''
+    Creates a dictionary with  each crossdomain data as key and an empty dataframe as value
+
+    Arguments:
+        single_neuron_image: boolean (True: add additional column with split number, False: no additional column)
+
+    Returns:
+        dictionary with  each crossdomain data as key and an empty dataframe as value. Columns are #Neurons, #Images training, Accuracy test data
+    '''
+    dataframe_dict = {}
+    # Create dataframe
+    if not single_neuron_image:
+        df = pd.DataFrame(columns=['#Neurons', '#Images training', 'Accuracy test data'])
+    else:
+        df = pd.DataFrame(columns=['#Neurons', '#Images training', 'Accuracy test data', 'Split number'])
+
+    crossdomains = ['hvm', 'cartoon', 'line_drawing', 'mosaic', 'painting', 'sketch', 'convex_hull', 'outline',
+                    'skeleton', 'silhouette', 'cococolor', 'cocogray', 'tdw']
+    for crossdomain in crossdomains:
+        dataframe_dict[crossdomain] = copy.copy(df)
+    return dataframe_dict
+
+
+def split_training_test_images(crossdomain_data_dictionary):
+    '''
+    Splits data into training data pool and test images. Make sure that background id of testing hvm and training non-hvm images are not identical.
+
+    Arguments:
+        crossdomain_data_dictionary (dict): dictionary which contains each crossdomain as key and respective NeuronRecordingAssembly as values
+
+    Returns:
+        crossdomain_test_data_dictionary (dict): dictionary with each crossdomain as key and respective test images (NeuronRecordingAssembly) as values
+        training_images (NeuronRecordingAssembly): training images pool. Contains only HVM images
+    '''
+    # Create crossdomain testing images dictionary
+    crossdomain_test_data_dictionary = {}
+    # Loop through each crossdomain and seed a random subset of 50 images for testing
+    for crossdomain in crossdomain_data_dictionary.keys():
+        crossdomain_data = crossdomain_data_dictionary[crossdomain]
+        if crossdomain == 'hvm':
+            test_images, training_images = reduce_data_num_images(data_complete=crossdomain_data,
+                                                                  number_images=HVM_TEST_IMAGES_NUM)
+            background_ids_silhouette_img = test_images.background_id.values
+
+        elif crossdomain in SILHOUETTE_DOMAINS:
+            test_indices = np.where(np.in1d(crossdomain_data.background_id, background_ids_silhouette_img))
+            test_images = crossdomain_data[test_indices]
+        else:
+            test_images, _ = reduce_data_num_images(data_complete=crossdomain_data, number_images=OOD_TEST_IMAGES_NUM)
+        crossdomain_test_data_dictionary[crossdomain] = test_images
+
+    return crossdomain_test_data_dictionary, training_images
+
+
+def get_decoder(data, estimator):
+    '''
+    Trains decoder.
+
+    Arguments:
+        data: (NeuronRecordingAssembly) xarray
+        estimator: (sklearn classifier function) Estimator e.g. RidgeClassifierCV, ElasticNetCV etc.
+
+    Returns:
+        clf: trained decoder
+    '''
+    # Get input & output data
+    X = data.data
+    try:
+        y = data.object_label.data
+    except:
+        y = data.category_name.data
+
+    # Get estimator
+    clf = copy.copy(estimator)  # Ridge Regression CV
+    clf.fit(X, y)
+
+    return clf
+
+
+def get_classifier_score_2AFC(classifier, data):
+    '''
+    Calculates the 2AFC score
+
+    Arguments:
+        classifier: pre-trained classifier
+        data (NeuronRecordingAssembly): test data
+
+    Returns:
+       2AFC score
+    '''
+    # Get input & output data
+    X = data.data
+    try:
+        y = data.object_label.data
+    except:
+        y = data.category_name.data
+
+    categories = np.unique(y)
+    number_of_categories = len(categories)
+    predict_probs = classifier.decision_function(X)
+    scores = np.zeros(len(y))
+    indices_row = np.arange(len(y))
+    indices_column = np.arange(len(categories))
+
+    for indx in indices_row:
+        category_index = np.where(categories == y[indx])
+        sum = 0
+        indx_column = np.delete(indices_column, category_index)
+        for idx in indx_column:
+            if predict_probs[indx, category_index] > predict_probs[indx, idx]:
+                sum = sum + 1
+            else:
+                continue
+
+        score = sum / (number_of_categories - 1)
+        scores[indx] = score
+
+    avg_score = np.mean(scores)
+
+    return avg_score
+
+
+def reduce_data_num_images(data_complete, number_images):
+    '''
+    Draws a randomly seeded subset of data while making sure that each object category is represented equally
+
+    Arguments:
+        data_complete (NeuronRecordingAssembly): complete dataset
+        number_images (int): number of images for training dataset
+
+    Returns:
+        stratified_training_data (NeuronRecordingAssembly): training data with equal number of each object category
+        rest_data (NeuronRecordingAssembly): remaining data from complete data - training data
+    '''
+    if number_images == len(data_complete):
+        place_holder = None
+        return data_complete, place_holder
+    else:
+        try:
+            stratified_training_data, rest_data = train_test_split(data_complete, train_size=number_images,
+                                                                   stratify=data_complete.object_label)
+        except:
+            stratified_training_data, rest_data = train_test_split(data_complete, train_size=number_images,
+                                                                   stratify=data_complete.category_name)
+
+        return stratified_training_data, rest_data

brainscore_vision/benchmarks/igustibagus2024/domain_transfer_neural.py

@@ -0,0 +1,134 @@
+# import brain-score specific libraries
+from brainio.assemblies import NeuronRecordingAssembly
+from brainscore_vision import load_dataset, load_ceiling, load_metric
+from brainscore_vision.benchmark_helpers.neural_common import NeuralBenchmark, average_repetition
+from brainscore_vision.utils import LazyLoad
+
+#### Define constants ####
+
+VISUAL_DEGREES = 8 
+NUMBER_OF_TRIALS = 63 
+BIBTEX = """
+
+"""
+
+ANIMALS = ['Oleo', 'Pico']
+
+CATEGORIES = ['apple', 'bear', 'bird', 'car', 'chair', 'dog', 'elephant', 'face', 'plane', 'zebra']
+SILHOUETTE_DOMAINS = ['convex_hull', 'outline', 'skeleton', 'silhouette']
+
+STYLES = ['hvm', 'convex_hull', 'outline', 'silhouette', 'cartoon', 'line_drawing', 'mosaic', 'painting', 'sketch']
+
+SPLIT_NUMBER = 100
+MAX_NUM_NEURONS = 71
+HVM_TEST_IMAGES_NUM = 30
+OOD_TEST_IMAGES_NUM = 30
+
+
+#### Benchmark implementation ####
+
+def Igustibagus2024_ridge():
+    assembly_repetition = LazyLoad(lambda: load_domain_transfer(average_repetitions=False))
+    assembly = LazyLoad(lambda: load_domain_transfer(average_repetitions=True))
+    similarity_metric = load_metric('ridge', crossvalidation_kwargs={'stratification_coord': 'object_label',
+                                                                     'preprocess_indices': _preprocess_indices})
+    ceiler=load_ceiling('internal_consistency')
+    return NeuralBenchmark(identifier=f'Igustibagus2024-ridge', version=1, parent='IT',
+                           assembly=assembly, similarity_metric=similarity_metric,
+                           visual_degrees=VISUAL_DEGREES, number_of_trials=NUMBER_OF_TRIALS,
+                           ceiling_func=lambda: ceiler(assembly_repetition),
+                           bibtex=BIBTEX)
+
+#### helpers ####
+
+def load_domain_transfer(average_repetitions):
+    '''
+     Loads the domain transfer data from local folders.
+
+     Returns:
+         assembly (NeuronRecordingAssembly): domain transfer data from pico
+    '''
+
+    assembly = load_dataset('Igustibagus2024')
+    assembly.load()
+    assembly = assembly.sel(time_bin_id=0)  # 70-170ms
+    assembly = assembly.squeeze('time_bin')
+    assembly = filter_neuroids(assembly=assembly, threshold=0.7) # filter neurons with low internal consistency
+
+    if average_repetitions:
+        assembly = average_repetition(assembly) 
+    
+    # Delete unwanted sources from the assembly and stimulus_set
+    include_sources = ['COCOColor', 'Art', 'COCOGray', 'TDW', 'Silhouette']
+    include_objects = ['silhouettes', 'edges', 'original', 'sketch', 'textures', 'painting', 'mosaic', 'grayscale', 'convex_hull', 'outline', 'silhouette', 'line_drawing', 'cartoon']
+    assembly = assembly.where(assembly.stimulus_source.isin(include_sources), drop=True)
+    assembly = assembly.where(assembly.object_style.isin(include_objects), drop=True)
+    stimulus_set = assembly.stimulus_set.copy()
+    filtered_stimulus_set = stimulus_set[stimulus_set.stimulus_source.isin(include_sources)]
+    filtered_stimulus_set = filtered_stimulus_set[filtered_stimulus_set.object_style.isin(include_objects)]
+    # 
+    assembly.attrs['stimulus_set']=filtered_stimulus_set
+    assembly = assembly.transpose('presentation', 'neuroid')
+    assembly = NeuronRecordingAssembly(assembly)
+    
+    return assembly
+
+
+def timebins_from_assembly(assembly):
+    timebins = assembly['time_bin'].values
+    if 'time_bin' not in assembly.dims:
+        timebins = [timebins]  # only single time-bin
+    return timebins
+
+def get_non_overlapping_indices(train_ids, test_ids):
+        """This function identifies non-overlapping background indices between train and test sets.
+
+        Arguments:
+        train_ids -- list: IDs in the training set
+        test_ids -- list: IDs in the test set
+
+        Returns:
+        train_indices_to_retain -- list: Indices of non-overlapping IDs in the training set
+        """
+
+        train_set = set(train_ids)
+        test_set = set(test_ids)
+
+        common_non_zero = train_set.intersection(test_set) - {0}
+        if not common_non_zero:
+            return train_ids, test_ids
+
+        # Find indices in the training set where background IDs are not common non-zero elements
+        train_indices_to_retain = [i for i, bg_id in enumerate(train_ids) if bg_id not in common_non_zero]
+
+        return train_indices_to_retain
+
+def _preprocess_indices(train_indices, test_indices, source_assembly):
+        """This function preprocesses indices for training and testing data.
+
+        Arguments:
+        train_indices -- array-like: Candidate indices for training data
+        test_indices -- array-like: Candidate indices for testing data
+        source_assembly -- DataFrame: Source data containing 'background_id' column
+
+        Returns:
+        train_indices -- array-like: Preprocessed training indices
+        test_indices -- array-like: Unmodified testing indices
+        """
+
+        train_background_ids = source_assembly['background_id'].values[train_indices]
+        test_background_ids = source_assembly['background_id'].values[test_indices]
+        train_indices = train_indices[get_non_overlapping_indices(train_background_ids, test_background_ids)]
+
+        assert set(source_assembly['background_id'].values[train_indices]).intersection(set( source_assembly['background_id'].values[test_indices])) == {0}
+        
+        return train_indices, test_indices
+
+def filter_neuroids(assembly, threshold):
+    ceiler = load_ceiling('internal_consistency')
+    ceiling = ceiler(assembly)
+    good_neuroid_indices = ceiling.attrs['raw'].mean('split') > threshold
+    assembly = assembly[{'neuroid': good_neuroid_indices.values}]
+    assembly = assembly.dropna(dim='presentation', how='all')
+    return assembly
+

brainscore_vision/benchmarks/igustibagus2024/test.py

@@ -0,0 +1,45 @@
+import numpy as np
+import pytest
+from numpy.random import RandomState
+from pytest import approx
+
+from brainio.assemblies import NeuroidAssembly
+from brainscore_vision import load_benchmark
+from brainscore_vision.benchmark_helpers import PrecomputedFeatures
+from brainscore_vision.benchmark_helpers.test_helper import StandardizedTests
+from brainscore_vision.benchmarks.igustibagus2024.domain_transfer_neural import load_domain_transfer
+
+
+@pytest.mark.memory_intense
+@pytest.mark.private_access
+def test_preprocessed_assembly():
+    assembly = load_domain_transfer(average_repetitions=True)
+    assert assembly.attrs['stimulus_set_identifier'] == 'Igustibagus2024'
+    assert len(assembly['presentation']) == 780
+    assert set(assembly['object_style'].values) == {'silhouette', 'sketch', 'cartoon', 'original', 'painting',
+                                                    'line_drawing', 'outline', 'convex_hull', 'mosaic'}
+    assert len(assembly['neuroid']) == 110
+
+
+@pytest.mark.private_access
+def test_self_regression():
+    standardized_tests = StandardizedTests()
+    standardized_tests.self_regression_test(benchmark='Igustibagus2024-ridge', visual_degrees=8,
+                                            expected=approx(1, abs=.005))
+
+
+@pytest.mark.private_access
+def test_engineering():
+    benchmark = load_benchmark('Igustibagus2024.IT_readout-accuracy')
+    rng = RandomState(0)
+    random_features = rng.random(size=(len(benchmark._stimuli), 300))
+    features = NeuroidAssembly(random_features, coords={**{
+        column: ('presentation', benchmark._stimuli[column]) for column in benchmark._stimuli
+    }, **{
+        'neuroid_num': ('neuroid', np.arange(random_features.shape[1])),
+        'neuroid_id': ('neuroid', np.arange(random_features.shape[1]))
+    }}, dims=['presentation', 'neuroid'])
+    features = PrecomputedFeatures(features, visual_degrees=8)
+    score = benchmark(features)
+    assert score.sel(aggregation='center') == approx(0.5, abs=0.02)  # chance
+    assert set(score.raw.dims) == {'domain', 'split'}

brainscore_vision/benchmarks/imagenet/__init__.py

@@ -0,0 +1,4 @@
+from brainscore_vision import benchmark_registry
+from .benchmark import Imagenet2012
+
+benchmark_registry['ImageNet-top1'] = Imagenet2012

brainscore_vision/benchmarks/imagenet/benchmark.py

@@ -0,0 +1,50 @@
+from pathlib import Path
+
+import pandas as pd
+
+from brainio.stimuli import StimulusSet, StimulusSetLoader
+from brainscore_core import Score
+from brainscore_vision import load_metric
+from brainscore_vision.benchmarks import BenchmarkBase
+from brainscore_vision.model_interface import BrainModel
+
+NUMBER_OF_TRIALS = 10
+
+
+class Imagenet2012(BenchmarkBase):
+    def __init__(self):
+        # Not a valid StimulusSet if the files are not present on the filesystem.
+        # We bypass StimulusSet.from_files() here.
+        stimulus_set = pd.read_csv(Path(__file__).parent / 'imagenet2012.csv')
+        StimulusSetLoader.correct_stimulus_id_name(stimulus_set)
+        stimulus_set = StimulusSet(stimulus_set)
+        stimulus_set.stimulus_paths = {row.stimulus_id: row.filepath for row in stimulus_set.itertuples()}
+        self._stimulus_set = stimulus_set
+        self._similarity_metric = load_metric('accuracy')
+        ceiling = Score(1)
+        super(Imagenet2012, self).__init__(identifier='ImageNet-top1', version=1,
+                                           ceiling_func=lambda: ceiling,
+                                           parent='ImageNet',
+                                           bibtex="""@INPROCEEDINGS{5206848,  
+                                                author={J. {Deng} and W. {Dong} and R. {Socher} and L. {Li} and  {Kai Li} and  {Li Fei-Fei}},  
+                                                booktitle={2009 IEEE Conference on Computer Vision and Pattern Recognition},   
+                                                title={ImageNet: A large-scale hierarchical image database},   
+                                                year={2009},  
+                                                volume={},  
+                                                number={},  
+                                                pages={248-255},
+                                                url = {https://ieeexplore.ieee.org/document/5206848}
+                                            }""")
+
+    def __call__(self, candidate):
+        # The proper `fitting_stimuli` to pass to the candidate would be the imagenet training set.
+        # For now, since almost all models in our hands were trained with imagenet, we'll just short-cut this
+        # by telling the candidate to use its pre-trained imagenet weights.
+        candidate.start_task(BrainModel.Task.label, 'imagenet')
+        stimulus_set = self._stimulus_set[list(set(self._stimulus_set.columns) - {'synset'})]  # do not show label
+        predictions = candidate.look_at(stimulus_set, number_of_trials=NUMBER_OF_TRIALS)
+        score = self._similarity_metric(
+            predictions.sortby('filename'),
+            self._stimulus_set.sort_values('filename')['synset'].values
+        )
+        return score